moving save_carefully()

This commit is contained in:
Philip Sargent 2021-04-13 00:11:08 +01:00
parent b602f3ae13
commit 2a1710596a
7 changed files with 54 additions and 53 deletions

View File

@ -32,7 +32,7 @@ class SimpleTest(SimpleTestCase):
from troggle.core.models_caves import Cave
def test_import_parsers_surveys(self):
from PIL import Image
from utils import save_carefully
from troggle.core.utils import save_carefully
from functools import reduce
def test_import_parsers_survex(self):
import troggle.settings as settings
@ -43,7 +43,6 @@ class SimpleTest(SimpleTestCase):
from troggle.core.views.caves import MapLocations
def test_import_parsers_QMs(self):
from troggle.core.models_caves import QM, Cave, LogbookEntry
from utils import save_carefully
def test_import_parsers_people(self):
from html.parser import HTMLParser
from unidecode import unidecode

View File

@ -60,6 +60,48 @@ def chaosmonkey(n):
# print("CHAOS strikes !", file=sys.stderr)
return True
def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
"""Looks up instance using lookupAttribs and carries out the following:
-if instance does not exist in DB: add instance to DB, return (new instance, True)
-if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
-if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)
The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
defined in core.models.TroggleModel.
"""
try:
instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
except:
print(" !! - SAVE CAREFULLY ===================", objectType)
print(" !! - -- objects.get_or_create()")
print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
raise
if not created and not instance.new_since_parsing:
for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title)
setattr(instance, k, v)
try:
instance.save()
except:
print(" !! - SAVE CAREFULLY ===================", objectType)
print(" !! - -- instance.save()")
print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
raise
try:
msg = str(instance)
except:
msg = "FAULT getting __str__ for instance with lookupattribs: {}:".format(lookupAttribs)
if created:
logging.info(str(instance) + ' was just added to the database for the first time. \n')
if not created and instance.new_since_parsing:
logging.info(str(instance) + " has been modified using Troggle, so the current script left it as is. \n")
if not created and not instance.new_since_parsing:
logging.info(str(instance) + " existed in the database unchanged since last parse. It was overwritten by the current script. \n")
return (instance, created)
# def get_related_by_wikilinks(wiki_text):
# found=re.findall(settings.QM_PATTERN,wiki_text)
# res=[]

View File

@ -7,7 +7,7 @@ from django.conf import settings
from troggle.core.models import DataIssue
from troggle.core.models_caves import QM, Cave, LogbookEntry
from utils import save_carefully
from troggle.core.utils import save_carefully
def deleteQMs():
@ -110,7 +110,7 @@ def parse_KH_QMs(kh, inputFile):
khQMs=open(os.path.join(settings.EXPOWEB, inputFile),'r')
khQMs=khQMs.readlines()
for line in khQMs:
res=re.search('name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABDCV])<dd>(?P<description>.*)\[(?P<nearest_station>.*)\]',line)
res=re.search(r'name=\"[CB](?P<year>\d*)-(?P<cave>\d*)-(?P<number>\d*).*</a> (?P<grade>[ABDCV])<dd>(?P<description>.*)\[(?P<nearest_station>.*)\]',line)
if res:
res=res.groupdict()
year=int(res['year'])

View File

@ -11,10 +11,9 @@ from django.template.defaultfilters import slugify
from django.utils.timezone import get_current_timezone, make_aware
from troggle.core.models import DataIssue, Expedition
from troggle.core.utils import TROG
from troggle.core.utils import TROG, save_carefully
from troggle.core.models_caves import Cave, LogbookEntry, PersonTrip, GetCaveLookup
from parsers.people import GetPersonExpeditionNameLookup
from utils import save_carefully
#
# When we edit logbook entries, allow a "?" after any piece of data to say we've frigged it and

View File

@ -1,11 +1,13 @@
from django.conf import settings
import troggle.core.models as models
import csv, re, datetime, os, shutil
from utils import save_carefully
from html.parser import HTMLParser
from unidecode import unidecode
'''These functions do not match how the stand-alone script works. So the script produces an HTML file which has
from django.conf import settings
import troggle.core.models as models
from troggle.core.utils import save_carefully
'''These functions do not match how the stand-alone folk script works. So the script produces an HTML file which has
href links to pages in troggle which troggle does not think are right.
The standalone script needs to be renedred defucnt, and all the parsing needs to be in troggle. Either that,
or they should use the same code by importing a module.

View File

@ -8,12 +8,12 @@ import re
import datetime
from PIL import Image
from utils import save_carefully
from functools import reduce
import settings
from troggle.core.models_survex import SingleScan, ScansFolder, TunnelFile
from troggle.core.models import DataIssue
from troggle.core.utils import save_carefully
def get_or_create_placeholder(year):

View File

@ -46,48 +46,7 @@ def randomLogbookSentence():
return randSent
def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
"""Looks up instance using lookupAttribs and carries out the following:
-if instance does not exist in DB: add instance to DB, return (new instance, True)
-if instance exists in DB and was modified using Troggle: do nothing, return (existing instance, False)
-if instance exists in DB and was not modified using Troggle: overwrite instance, return (instance, False)
The checking is accomplished using Django's get_or_create and the new_since_parsing boolean field
defined in core.models.TroggleModel.
"""
try:
instance, created = objectType.objects.get_or_create(defaults=nonLookupAttribs, **lookupAttribs)
except:
print(" !! - SAVE CAREFULLY ===================", objectType)
print(" !! - -- objects.get_or_create()")
print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
raise
if not created and not instance.new_since_parsing:
for k, v in list(nonLookupAttribs.items()): #overwrite the existing attributes from the logbook text (except date and title)
setattr(instance, k, v)
try:
instance.save()
except:
print(" !! - SAVE CAREFULLY ===================", objectType)
print(" !! - -- instance.save()")
print(" !! - lookupAttribs:{}\n !! - nonLookupAttribs:{}".format(lookupAttribs,nonLookupAttribs))
raise
try:
msg = str(instance)
except:
msg = "FAULT getting __str__ for instance with lookupattribs: {}:".format(lookupAttribs)
if created:
logging.info(str(instance) + ' was just added to the database for the first time. \n')
if not created and instance.new_since_parsing:
logging.info(str(instance) + " has been modified using Troggle, so the current script left it as is. \n")
if not created and not instance.new_since_parsing:
logging.info(str(instance) + " existed in the database unchanged since last parse. It was overwritten by the current script. \n")
return (instance, created)
re_body = re.compile(r"\<body[^>]*\>(.*)\</body\>", re.DOTALL)
re_title = re.compile(r"\<title[^>]*\>(.*)\</title\>", re.DOTALL)