expo/troggle
2
0
Fork 1
mirror of https://expo.survex.com/repositories/troggle/.git synced 2024-11-22 15:21:52 +00:00
Code Issues 19 Projects 3 Activity

Make the suryeys importer not explode

Browse Source
This commit is contained in:
Sam Wenham 2019-02-24 14:29:14 +00:00
parent 8f66837f6f
commit 8fd23008e3
3 changed files with 68 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
databaseReset.py
View File

@ -22,7 +22,7 @@ def reload_db():
os.remove(databasename) os.remove(databasename)
except OSError: except OSError:
pass pass
else: else:
cursor = connection.cursor() cursor = connection.cursor()
cursor.execute("DROP DATABASE %s" % databasename) cursor.execute("DROP DATABASE %s" % databasename)
cursor.execute("CREATE DATABASE %s" % databasename) cursor.execute("CREATE DATABASE %s" % databasename)
@ -115,7 +115,7 @@ def import_auto_logbooks():
"autologbook") "autologbook")
for root, dirs, filenames in os.walk(directory): for root, dirs, filenames in os.walk(directory):
for filename in filenames: for filename in filenames:
print os.path.join(root, filename) print(os.path.join(root, filename))
parsers.logbooks.parseAutoLogBookEntry(os.path.join(root, filename)) parsers.logbooks.parseAutoLogBookEntry(os.path.join(root, filename))
#Temporary function until definative source of data transfered. #Temporary function until definative source of data transfered.
@ -138,7 +138,7 @@ def dumplogbooks():
filename = os.path.join(directory, filename = os.path.join(directory,
dateStr + "." + slugify(lbe.title)[:50] + ".html") dateStr + "." + slugify(lbe.title)[:50] + ".html")
if lbe.cave: if lbe.cave:
print lbe.cave.reference() print(lbe.cave.reference())
trip = {"title": lbe.title, "html":lbe.text, "cave": lbe.cave.reference(), "caveOrLocation": "cave"} trip = {"title": lbe.title, "html":lbe.text, "cave": lbe.cave.reference(), "caveOrLocation": "cave"}
else: else:
trip = {"title": lbe.title, "html":lbe.text, "location":lbe.place, "caveOrLocation": "location"} trip = {"title": lbe.title, "html":lbe.text, "location":lbe.place, "caveOrLocation": "location"}
@ -180,6 +180,7 @@ def usage():
scans - read in the scanned surveynotes scans - read in the scanned surveynotes
survex - read in the survex files survex - read in the survex files
survexpos survexpos
surveys
tunnel - read in the Tunnel files tunnel - read in the Tunnel files
writeCaves writeCaves
""") """)
@ -212,7 +213,7 @@ if __name__ == "__main__":
try: try:
import_tunnelfiles() import_tunnelfiles()
except: except:
print "Tunnel files parser broken." print("Tunnel files parser broken.")
import_surveys() import_surveys()
import_descriptions() import_descriptions()
parse_descriptions() parse_descriptions()
@ -232,6 +233,8 @@ if __name__ == "__main__":
dumplogbooks() dumplogbooks()
elif "writeCaves" in sys.argv: elif "writeCaves" in sys.argv:
writeCaves() writeCaves()
elif "surveys" in sys.argv:
import_surveys()
elif "help" in sys.argv: elif "help" in sys.argv:
usage() usage()
else: else:

2
docker/requirements.txt.dj-1.8.19
View File

@ -1,7 +1,7 @@
Django==1.8.19 Django==1.8.19
django-registration==2.1.2 django-registration==2.1.2
mysql mysql
imagekit django-imagekit
Image Image
django-tinymce==2.7.0 django-tinymce==2.7.0
smartencoding smartencoding

111
parsers/surveys.py
View File

@ -39,7 +39,7 @@ def readSurveysFromCSV():
# test if the expeditions have been added yet # test if the expeditions have been added yet
if Expedition.objects.count()==0: if Expedition.objects.count()==0:
print "There are no expeditions in the database. Please run the logbook parser." print("There are no expeditions in the database. Please run the logbook parser.")
sys.exit() sys.exit()
@ -56,7 +56,7 @@ def readSurveysFromCSV():
for survey in surveyreader: for survey in surveyreader:
#I hate this, but some surveys have a letter eg 2000#34a. The next line deals with that. #I hate this, but some surveys have a letter eg 2000#34a. The next line deals with that.
walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']]) walletNumberLetter = re.match(r'(?P<number>\d*)(?P<letter>[a-zA-Z]*)',survey[header['Survey Number']])
# print walletNumberLetter.groups() # print(walletNumberLetter.groups())
year=survey[header['Year']] year=survey[header['Year']]
@ -89,63 +89,72 @@ def listdir(*directories):
# add survey scans # add survey scans
def parseSurveyScans(expedition, logfile=None): def parseSurveyScans(expedition, logfile=None):
# yearFileList = listdir(expedition.year) # yearFileList = listdir(expedition.year)
yearPath=os.path.join(settings.SURVEY_SCANS, "surveyscans", expedition.year) try:
yearFileList=os.listdir(yearPath) yearPath=os.path.join(settings.SURVEY_SCANS, "surveyscans", expedition.year)
print yearFileList yearFileList=os.listdir(yearPath)
for surveyFolder in yearFileList: print(yearFileList)
try: for surveyFolder in yearFileList:
surveyNumber=re.match(r'\d\d\d\d#0*(\d+)',surveyFolder).groups()
# scanList = listdir(expedition.year, surveyFolder)
scanList=os.listdir(os.path.join(yearPath,surveyFolder))
except AttributeError:
print surveyFolder + " ignored\r",
continue
for scan in scanList:
try: try:
scanChopped=re.match(r'(?i).*(notes|elev|plan|elevation|extend)(\d*)\.(png|jpg|jpeg)',scan).groups() surveyNumber=re.match(r'\d\d\d\d#0*(\d+)',surveyFolder).groups()
scanType,scanNumber,scanFormat=scanChopped #scanList = listdir(expedition.year, surveyFolder)
scanList=os.listdir(os.path.join(yearPath,surveyFolder))
except AttributeError: except AttributeError:
print scan + " ignored\r", print(surveyFolder + " ignored\r",)
continue continue
if scanType == 'elev' or scanType == 'extend':
scanType = 'elevation'
if scanNumber=='': for scan in scanList:
scanNumber=1 try:
scanChopped=re.match(r'(?i).*(notes|elev|plan|elevation|extend)(\d*)\.(png|jpg|jpeg)',scan).groups()
scanType,scanNumber,scanFormat=scanChopped
except AttributeError:
print(scan + " ignored\r",)
continue
if scanType == 'elev' or scanType == 'extend':
scanType = 'elevation'
if type(surveyNumber)==types.TupleType: if scanNumber=='':
surveyNumber=surveyNumber[0] scanNumber=1
try:
placeholder=get_or_create_placeholder(year=int(expedition.year)) if type(surveyNumber)==types.TupleType:
survey=Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=expedition, defaults={'logbook_entry':placeholder})[0] surveyNumber=surveyNumber[0]
except Survey.MultipleObjectsReturned: try:
survey=Survey.objects.filter(wallet_number=surveyNumber, expedition=expedition)[0] placeholder=get_or_create_placeholder(year=int(expedition.year))
file_=os.path.join(yearPath, surveyFolder, scan) survey=Survey.objects.get_or_create(wallet_number=surveyNumber, expedition=expedition, defaults={'logbook_entry':placeholder})[0]
scanObj = ScannedImage( except Survey.MultipleObjectsReturned:
file=file_, survey=Survey.objects.filter(wallet_number=surveyNumber, expedition=expedition)[0]
contents=scanType, file_=os.path.join(yearPath, surveyFolder, scan)
number_in_wallet=scanNumber, scanObj = ScannedImage(
survey=survey, file=file_,
new_since_parsing=False, contents=scanType,
) number_in_wallet=scanNumber,
print "Added scanned image at " + str(scanObj) survey=survey,
#if scanFormat=="png": new_since_parsing=False,
#if isInterlacedPNG(os.path.join(settings.SURVEY_SCANS, "surveyscans", file_)): )
# print file_+ " is an interlaced PNG. No can do." print("Added scanned image at " + str(scanObj))
#continue #if scanFormat=="png":
scanObj.save() #if isInterlacedPNG(os.path.join(settings.SURVEY_SCANS, "surveyscans", file_)):
# print file_+ " is an interlaced PNG. No can do."
#continue
scanObj.save()
except (IOError, OSError):
yearPath=os.path.join(settings.SURVEY_SCANS, "surveyscans", expedition.year)
print("No folder found for " + expedition.year + " at:- " + yearPath)
# dead # dead
def parseSurveys(logfile=None): def parseSurveys(logfile=None):
readSurveysFromCSV() try:
readSurveysFromCSV()
except (IOError, OSError):
print("Survey CSV not found..")
pass
for expedition in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then for expedition in Expedition.objects.filter(year__gte=2000): #expos since 2000, because paths and filenames were nonstandard before then
parseSurveyScans(expedition) parseSurveyScans(expedition)
# dead # dead
def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL) def isInterlacedPNG(filePath): #We need to check for interlaced PNGs because the thumbnail engine can't handle them (uses PIL)
file=Image.open(filePath) file=Image.open(filePath)
print filePath print(filePath)
if 'interlace' in file.info: if 'interlace' in file.info:
return file.info['interlace'] return file.info['interlace']
else: else:
@ -181,7 +190,7 @@ def LoadListScansFile(survexscansfolder):
for (fyf, ffyf, fisdiryf) in gld: for (fyf, ffyf, fisdiryf) in gld:
assert not fisdiryf, ffyf assert not fisdiryf, ffyf
if re.search("\.(?:png|jpg|jpeg)(?i)$", fyf): if re.search(r"\.(?:png|jpg|jpeg)(?i)$", fyf):
survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder) survexscansingle = SurvexScanSingle(ffile=ffyf, name=fyf, survexscansfolder=survexscansfolder)
survexscansingle.save() survexscansingle.save()
@ -190,7 +199,7 @@ def LoadListScansFile(survexscansfolder):
# and builds up the models we can access later # and builds up the models we can access later
def LoadListScans(): def LoadListScans():
print 'Loading Survey Scans...' print('Loading Survey Scans...')
SurvexScanSingle.objects.all().delete() SurvexScanSingle.objects.all().delete()
SurvexScansFolder.objects.all().delete() SurvexScansFolder.objects.all().delete()
@ -208,7 +217,7 @@ def LoadListScans():
continue continue
# do the year folders # do the year folders
if re.match("\d\d\d\d$", f): if re.match(r"\d\d\d\d$", f):
for fy, ffy, fisdiry in GetListDir(ff): for fy, ffy, fisdiry in GetListDir(ff):
if fisdiry: if fisdiry:
assert fisdiry, ffy assert fisdiry, ffy
@ -225,7 +234,7 @@ def LoadListScans():
def FindTunnelScan(tunnelfile, path): def FindTunnelScan(tunnelfile, path):
scansfolder, scansfile = None, None scansfolder, scansfile = None, None
mscansdir = re.search("(\d\d\d\d#\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg))$", path) mscansdir = re.search(r"(\d\d\d\d#\d+\w?|1995-96kh|92-94Surveybookkh|1991surveybook|smkhs)/(.*?(?:png|jpg))$", path)
if mscansdir: if mscansdir:
scansfolderl = SurvexScansFolder.objects.filter(walletname=mscansdir.group(1)) scansfolderl = SurvexScansFolder.objects.filter(walletname=mscansdir.group(1))
if len(scansfolderl): if len(scansfolderl):
@ -242,9 +251,9 @@ def FindTunnelScan(tunnelfile, path):
if scansfile: if scansfile:
tunnelfile.survexscans.add(scansfile) tunnelfile.survexscans.add(scansfile)
elif path and not re.search("\.(?:png|jpg)$(?i)", path): elif path and not re.search(r"\.(?:png|jpg)$(?i)", path):
name = os.path.split(path)[1] name = os.path.split(path)[1]
print "ttt", tunnelfile.tunnelpath, path, name print("ttt", tunnelfile.tunnelpath, path, name)
rtunnelfilel = TunnelFile.objects.filter(tunnelname=name) rtunnelfilel = TunnelFile.objects.filter(tunnelname=name)
if len(rtunnelfilel): if len(rtunnelfilel):
assert len(rtunnelfilel) == 1, ("two paths with name of", path, "need more discrimination coded") assert len(rtunnelfilel) == 1, ("two paths with name of", path, "need more discrimination coded")