forked from expo/troggle
moving save_carefully()
This commit is contained in:
parent
b602f3ae13
commit
2a1710596a
@ -32,7 +32,7 @@ class SimpleTest(SimpleTestCase):
|
||||
from troggle.core.models_caves import Cave
|
||||
def test_import_parsers_surveys(self):
|
||||
from PIL import Image
|
||||
from utils import save_carefully
|
||||
from troggle.core.utils import save_carefully
|
||||
from functools import reduce
|
||||
def test_import_parsers_survex(self):
|
||||
import troggle.settings as settings
|
||||
@ -43,7 +43,6 @@ class SimpleTest(SimpleTestCase):
|
||||
from troggle.core.views.caves import MapLocations
|
||||
def test_import_parsers_QMs(self):
|
||||
from troggle.core.models_caves import QM, Cave, LogbookEntry
|
||||
from utils import save_carefully
|
||||
def test_import_parsers_people(self):
|
||||
from html.parser import HTMLParser
|
||||
from unidecode import unidecode
|
||||
|
@ -60,6 +60,48 @@ def chaosmonkey(n):
|
||||
# print("CHAOS strikes !", file=sys.stderr)
|
||||
return True
|
||||
|
||||
|
||||
def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
|
||||
"""Looks up instance using lookupAttribs and carries out the following:
|
||||
-if instance does not exist in DB: add instance to DB, return (new instance, True)
|
||||
-if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
|
||||
-if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)
|
||||
|
||||
The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
|
||||
defined in core.models.TroggleModel.
|
||||
|
||||
"""
|
||||
try:
|
||||
instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
|
||||
except:
|
||||
print(" !! - SAVE CAREFULLY ===================", objectType)
|
||||
print(" !! - -- objects.get_or_create()")
|
||||
print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
|
||||
raise
|
||||
if not created and not instance.new_since_parsing:
|
||||
for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title)
|
||||
setattr(instance, k, v)
|
||||
try:
|
||||
instance.save()
|
||||
except:
|
||||
print(" !! - SAVE CAREFULLY ===================", objectType)
|
||||
print(" !! - -- instance.save()")
|
||||
print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
|
||||
raise
|
||||
try:
|
||||
msg = str(instance)
|
||||
except:
|
||||
msg = "FAULT getting __str__ for instance with lookupattribs: {}:".format(lookupAttribs)
|
||||
if created:
|
||||
logging.info(str(instance) + ' was just added to the database for the first time. \n')
|
||||
|
||||
if not created and instance.new_since_parsing:
|
||||
logging.info(str(instance) + " has been modified using Troggle, so the current script left it as is. \n")
|
||||
|
||||
if not created and not instance.new_since_parsing:
|
||||
logging.info(str(instance) + " existed in the database unchanged since last parse. It was overwritten by the current script. \n")
|
||||
return (instance, created)
|
||||
|
||||
# def get_related_by_wikilinks(wiki_text):
|
||||
# found=re.findall(settings.QM_PATTERN,wiki_text)
|
||||
# res=[]
|
||||
|
@ -7,7 +7,7 @@ from django.conf import settings
|
||||
|
||||
from troggle.core.models import DataIssue
|
||||
from troggle.core.models_caves import QM, Cave, LogbookEntry
|
||||
from utils import save_carefully
|
||||
from troggle.core.utils import save_carefully
|
||||
|
||||
|
||||
def deleteQMs():
|
||||
@ -110,7 +110,7 @@ def parse_KH_QMs(kh, inputFile):
|
||||
khQMs=open(os.path.join(settings.EXPOWEB, inputFile),'r')
|
||||
khQMs=khQMs.readlines()
|
||||
for line in khQMs:
|
||||
res=re.search('name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABDCV])<dd>(?P<description>.*)\[(?P<nearest_station>.*)\]',line)
|
||||
res=re.search(r'name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABDCV])<dd>(?P<description>.*)\[(?P<nearest_station>.*)\]',line)
|
||||
if res:
|
||||
res=res.groupdict()
|
||||
year=int(res['year'])
|
||||
|
@ -11,10 +11,9 @@ from django.template.defaultfilters import slugify
|
||||
from django.utils.timezone import get_current_timezone, make_aware
|
||||
|
||||
from troggle.core.models import DataIssue, Expedition
|
||||
from troggle.core.utils import TROG
|
||||
from troggle.core.utils import TROG, save_carefully
|
||||
from troggle.core.models_caves import Cave, LogbookEntry, PersonTrip, GetCaveLookup
|
||||
from parsers.people import GetPersonExpeditionNameLookup
|
||||
from utils import save_carefully
|
||||
|
||||
#
|
||||
# When we edit logbook entries, allow a "?" after any piece of data to say we've frigged it and
|
||||
|
@ -1,11 +1,13 @@
|
||||
from django.conf import settings
|
||||
import troggle.core.models as models
|
||||
import csv, re, datetime, os, shutil
|
||||
from utils import save_carefully
|
||||
from html.parser import HTMLParser
|
||||
from unidecode import unidecode
|
||||
|
||||
'''These functions do not match how the stand-alone script works. So the script produces an HTML file which has
|
||||
from django.conf import settings
|
||||
|
||||
import troggle.core.models as models
|
||||
from troggle.core.utils import save_carefully
|
||||
|
||||
'''These functions do not match how the stand-alone folk script works. So the script produces an HTML file which has
|
||||
href links to pages in troggle which troggle does not think are right.
|
||||
The standalone script needs to be renedred defucnt, and all the parsing needs to be in troggle. Either that,
|
||||
or they should use the same code by importing a module.
|
||||
|
@ -8,12 +8,12 @@ import re
|
||||
import datetime
|
||||
|
||||
from PIL import Image
|
||||
from utils import save_carefully
|
||||
from functools import reduce
|
||||
|
||||
import settings
|
||||
from troggle.core.models_survex import SingleScan, ScansFolder, TunnelFile
|
||||
from troggle.core.models import DataIssue
|
||||
from troggle.core.utils import save_carefully
|
||||
|
||||
|
||||
def get_or_create_placeholder(year):
|
||||
|
@ -46,48 +46,7 @@ def randomLogbookSentence():
|
||||
|
||||
return randSent
|
||||
|
||||
|
||||
def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
|
||||
"""Looks up instance using lookupAttribs and carries out the following:
|
||||
-if instance does not exist in DB: add instance to DB, return (new instance, True)
|
||||
-if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
|
||||
-if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)
|
||||
|
||||
The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
|
||||
defined in core.models.TroggleModel.
|
||||
|
||||
"""
|
||||
try:
|
||||
instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
|
||||
except:
|
||||
print(" !! - SAVE CAREFULLY ===================", objectType)
|
||||
print(" !! - -- objects.get_or_create()")
|
||||
print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
|
||||
raise
|
||||
if not created and not instance.new_since_parsing:
|
||||
for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title)
|
||||
setattr(instance, k, v)
|
||||
try:
|
||||
instance.save()
|
||||
except:
|
||||
print(" !! - SAVE CAREFULLY ===================", objectType)
|
||||
print(" !! - -- instance.save()")
|
||||
print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
|
||||
raise
|
||||
try:
|
||||
msg = str(instance)
|
||||
except:
|
||||
msg = "FAULT getting __str__ for instance with lookupattribs: {}:".format(lookupAttribs)
|
||||
if created:
|
||||
logging.info(str(instance) + ' was just added to the database for the first time. \n')
|
||||
|
||||
if not created and instance.new_since_parsing:
|
||||
logging.info(str(instance) + " has been modified using Troggle, so the current script left it as is. \n")
|
||||
|
||||
if not created and not instance.new_since_parsing:
|
||||
logging.info(str(instance) + " existed in the database unchanged since last parse. It was overwritten by the current script. \n")
|
||||
return (instance, created)
|
||||
|
||||
|
||||
re_body = re.compile(r"\<body[^>]*\>(.*)\</body\>", re.DOTALL)
|
||||
re_title = re.compile(r"\<title[^>]*\>(.*)\</title\>", re.DOTALL)
|
||||
|
Loading…
x
Reference in New Issue
Block a user