From ccfc44a4234e3fb6b590d79fa76f7e03b6d5dca0 Mon Sep 17 00:00:00 2001 From: Philip Sargent Date: Mon, 6 Mar 2023 04:52:41 +0000 Subject: [PATCH] Saving new survex file parses contents. --- core/models/survex.py | 4 +- core/views/survex.py | 57 ++++++++++++--------- parsers/survex.py | 115 ++++++++++++++++++++++++------------------ 3 files changed, 102 insertions(+), 74 deletions(-) diff --git a/core/models/survex.py b/core/models/survex.py index c6d6af1..c1d8ca2 100644 --- a/core/models/survex.py +++ b/core/models/survex.py @@ -116,7 +116,9 @@ class SurvexBlockLookUpManager(models.Manager): class SurvexBlock(models.Model): - """One begin..end block within a survex file. The basic element of a survey trip.""" + """One begin..end block within a survex file. The basic element of a survey trip. + Multiple anonymous survex blocks are possible within the same surfex file + """ objects = SurvexBlockLookUpManager() name = models.CharField(max_length=100) diff --git a/core/views/survex.py b/core/views/survex.py index e60df6e..ea00fcc 100644 --- a/core/views/survex.py +++ b/core/views/survex.py @@ -38,7 +38,7 @@ even though there are dozens of surveys. - the primary survex file in each cave directory should be in a configuration, not buried in the code... -- Save the edited survexfile as a survexfile object and re-parse it, and update +- Save and re-parse an edited survexfile which already exists in the db, and update all its dependencies (work in progress) """ @@ -115,6 +115,24 @@ survextemplatefile = """; *** THIS IS A TEMPLATE FILE NOT WHAT YOU MIGHT BE EXPE *end [surveyname] """ +def get_survexfile(filename): + """Gets the SurvexFile object from the survex path for the file + in a robust way + """ + refs = SurvexFile.objects.filter(path=filename) + if len(refs)==0: # new survex file, not created in db yet + survexfile = False + elif len(refs)==1: + survexfile = SurvexFile.objects.get(path=filename) + else: + survexfile = refs[0] + # OK this is due to a bug in the import file parsing, whoops. Now fixed ?! + print("BUG - to be fixed in the survex parser - not critical..") + print(f"Number of SurvexFile objects found: {len(refs)}") + for s in refs: + print (s.path, s.survexdirectory, s.cave) + # print(type(survexfile), filename) + return survexfile class SvxForm(forms.Form): """Two-pane form, upper half is the raw survex file, lower half (with green background) @@ -138,18 +156,8 @@ class SvxForm(forms.Form): self.template = True self.survexfile = False return survextemplatefile - refs = SurvexFile.objects.filter(path=self.data["filename"]) - if len(refs)==0: # new survex file, not created in db yet - self.survexfile = False - elif len(refs)==1: - self.survexfile = SurvexFile.objects.get(path=self.data["filename"]) - else: - self.survexfile = refs[0] - # OK this is due to a bug in the import file parsing, whoops. - print("BUG - to be fixed in the survex parser - not critical..") - print(f"Number of SurvexFile objects found: {len(refs)}") - for s in refs: - print (s.path, s.survexdirectory, s.cave) + if not self.survexfile: + self.survexfile = get_survexfile(self.data["filename"]) try: fin = open(fname, "r", encoding="utf8", newline="") svxtext = fin.read() @@ -205,10 +213,8 @@ class SvxForm(forms.Form): comment = f"Online survex edit: {self.data['filename']}.svx on dev machine '{socket.gethostname()}' " only_commit(fname, comment) - parse_one_file(self.data["filename"]) - return "SAVED and committed to git (if there were differences)" def Process(self): @@ -260,6 +266,8 @@ def svx(request, survex_file): Originally the non-existence of difflist was used as a marker to say that the file had been saved and that thuis there were no differences. This is inadequate, as a new file which has not been saved also has no difflist. + + Needs refactoring. Too many piecemeal edits and odd state dependencies. """ warning = False @@ -281,8 +289,11 @@ def svx(request, survex_file): rcode = rform.cleaned_data["code"] outputtype = rform.cleaned_data["outputtype"] # used by CodeMirror ajax I think difflist = form.DiffCode(rcode) - # print(">>>> ", rform.data) - svxfile = form.survexfile + # keys = [] + # for key in rform.data: + # keys.append(key) + # print(">>>> ", keys) + sfile = form.survexfile if "revert" in rform.data: pass @@ -291,7 +302,7 @@ def svx(request, survex_file): if difflist: message = "SAVE FILE FIRST" form.data["code"] = rcode - elif svxfile: + elif sfile: logmessage = form.Process() if logmessage: message = f"OUTPUT FROM PROCESSING\n{logmessage}" @@ -305,13 +316,14 @@ def svx(request, survex_file): message = "You do not have authority to save this file. Please log in." if message != "SAVED": form.data["code"] = rcode + if "diff" in rform.data: print("Differences: ") form.data["code"] = rcode - # GET or after POST-specific handling - svxfile = form.survexfile # only valid once form.GetDiscCode() called - + # GET, also fall-through after POST-specific handling + svxfile = get_survexfile(survex_file) + if "code" not in form.data: form.data["code"] = form.GetDiscCode() if form.template: @@ -324,10 +336,8 @@ def svx(request, survex_file): if message: difflist.insert(0, message) - # print [ form.data['code'] ] svxincludes = re.findall(r"(?i)\*include\s+(\S+)", form.data["code"] or "") - # collect all the survex blocks which actually have a valid date if svxfile: try: @@ -359,7 +369,6 @@ def svx(request, survex_file): "form": form, "events": events, } - # vmap.update(csrf(request)) # this now refreshes to the wrong value, now that we user render(request, if outputtype == "ajax": # used by CodeMirror ajax I think return render(request, "svxfiledifflistonly.html", vmap) diff --git a/parsers/survex.py b/parsers/survex.py index 57746e0..42c8c6b 100644 --- a/parsers/survex.py +++ b/parsers/survex.py @@ -333,7 +333,7 @@ class LoadingSurvex: currentteam = set() inheritteam = set() pending = [] - nocreate = False + adhocload = False def __init__(self): self.caveslist = GetCaveLookup() @@ -1253,10 +1253,6 @@ class LoadingSurvex: svxid = included.groups()[0] if svxid.lower() == debugprinttrigger.lower(): debugprint = True - if self.nocreate: - # skip *include files if we are parsing just one file ?! - # Maybe do this as a setting in the survexfile SAVE form? - return self.LoadSurvexFile(svxid) self.stacksvxfiles.append(self.currentsurvexfile) @@ -2138,19 +2134,24 @@ def FindAndLoadSurvex(survexblockroot): def parse_one_file(fpath): # --------------------------------------in progress------------------- """Parse just one file. Use when re-loading after editing. - NB careful of *include lines, which may exist! Ignore them. - WORK IN PROGRESS. NONE OF THIS WORKS. - Currently just returns True without saving anything - Problems with re-setting multiple anonymous blocks in the survexfile - Need to delete them all and recreate all the links, e.g. to wallets and people + NOTE: *include lines are ignored. + In the initial file parsing in databaseReset, the *include expansion is done + in an earlier stange than LinearLoad(). By the time LinearLoad() is called, + all the *include expansion has happened. + + WORK IN PROGRESS. + Works fine for completely new survex file. + + For an edited, pre-existing survex file, + deleting all the survex blocks means that some other pages just crash, e.g. + /expedition/1996 """ print(f"\n - Loading One Survex file '{fpath}'", file=sys.stderr) svx_load = LoadingSurvex() - collatefilename = Path(settings.SURVEX_DATA, (fpath + ".svx")) - print(f" - {collatefilename=}") - + fname = Path(settings.SURVEX_DATA, (fpath + ".svx")) + # print(f" - {fname=}") svxs = SurvexFile.objects.filter(path=fpath) # If this SurvexFile object already exists in the database, we want to keep the parent survexblock @@ -2159,51 +2160,66 @@ def parse_one_file(fpath): # --------------------------------------in progress-- # and we are not processing any *include we find if svxs: if len(svxs)>1: - print(f" ! Error. More than one survex file object in database with the same file-path {svx}") - raise - print(f" - pre-existing survexfile {svxs}") - + print(f" ! Mistake? More than one survex file object in database with the same file-path {svxs}") + print(f" - Aborting file parsing & import into database.") + return True + print(f" - Pre-existing survexfile {svxs}. NOT re-parsing now.") + # SurvexBlock.objects.all().delete() + + return True # NOT WORKING YET svx = svxs[0] # first and only member of QuerySet b = SurvexBlock.objects.filter(survexfile=svx) - if len(b)>1: - print(f" ! Error. More than one survex file object in database attached to survexblock {b}") - raise - - survexblockroot=b[0] - print(f" - {survexblockroot=}") + if len(b) >= 1: + # survexblockparent=b[0].parent + # survexblockparent.survexfile = svx + # # Stamp all over the accumulated lengths and legs in the parent block, + # # This also obliterates survey lengths from all other 'sibling' survex files + # # to the one being re-parsed + # survexblockparent.legsall=0 + # survexblockparent.legslength=0.0 + + b.delete() # deletes all pre-existing SurvexBlocks attached to this SurvexFile + # all these foreign keys should be recreated properly when the file is parsed. + # so whiy is /expedition/1996 crashing in nasty template error? + else: + survexblockparent = SurvexBlock( + name="fresh_parent", survexpath="", survexfile=svx, legsall=0, legslength=0.0 + ) + survexblockparent.save() + survexblockparent = SurvexBlock( + name="fresh_parent", survexpath="", survexfile=svx, legsall=0, legslength=0.0 + ) + survexblockparent.save() + print(f" - {survexblockparent=}") dir = svx.survexdirectory svx_load.survexdict[dir] = [svx] svx_load.svxdirs[""] = dir - return True - svx_load.nocreate = True + + svx_load.adhocload = True # ---------------------------------------------------------------- - svx_load.LinearLoad(survexblockroot, fpath, collatefilename) + svx_load.LinearLoad(survexblockparent, fpath, fname) # ---------------------------------------------------------------- + svx_load.adhocload = False else: - print(f" - Not seen this survexfile before '{fpath}'") + print(f" - Not seen this survexfile before '{fpath}' Loading...") - return True - omitsfileroot = MakeOmitFileRoot(fpath) # NO! This always creats a SurvexFile object. We are overwriting one.. - survexomitsroot = SurvexBlock( - name=OMITBLOCK, survexpath="", survexfile=omitsfileroot, legsall=0, legslength=0.0 - ) - survexomitsroot.save() - # SurvexBlock.objects.all().delete() - # SurvexFile.objects.all().delete() - # SurvexDirectory.objects.all().delete() - # SurvexPersonRole.objects.all().delete() - # SurvexStation.objects.all().delete() - print(" - Loading Survex Blocks...") - - svx_load.survexdict[survexfileroot.survexdirectory] = [] - svx_load.survexdict[survexfileroot.survexdirectory].append(survexfileroot) - svx_load.svxdirs[""] = survexfileroot.survexdirectory - - # ---------------------------------------------------------------- - svx_load.LinearLoad(survexblockroot, survexfileroot.path, collatefilename) - # ---------------------------------------------------------------- + newfileroot = MakeFileRoot(fpath) + survexblockparent = SurvexBlock( + name="adhoc_parent", survexpath="", survexfile=newfileroot, legsall=0, legslength=0.0 + ) + survexblockparent.save() + + svx_load.survexdict[newfileroot.survexdirectory] = [] + svx_load.survexdict[newfileroot.survexdirectory].append(newfileroot) + svx_load.svxdirs[""] = newfileroot.survexdirectory + + svx_load.adhocload = True + # ---------------------------------------------------------------- + svx_load.LinearLoad(survexblockparent, newfileroot.path, fname) + # ---------------------------------------------------------------- + svx_load.adhocload = False legsnumber = svx_load.legsnumber @@ -2213,6 +2229,7 @@ def parse_one_file(fpath): # --------------------------------------in progress-- tf += len(svx_load.survexdict[d]) print(f" - Number of SurvexFiles: {tf:,}") print(f" - Number of Survex legs: {legsnumber:,}") + print(f" - Length of Survex legs: {svx_load.slength}") # Now set Django SurvexFile object.. @@ -2234,7 +2251,7 @@ def MakeSurvexFileRoot(): return fileroot -def MakeOmitFileRoot(fn): +def MakeFileRoot(fn): """Returns a file_object.path = _unseens.svx associated with directory_object.path = SURVEX_DATA""" fileroot = SurvexFile(path=fn, cave=None) fileroot.survexdirectory = SurvexDirectory.objects.get(path=settings.SURVEX_DATA) @@ -2288,7 +2305,7 @@ def LoadSurvexBlocks(): # sudo service mariadb start survexblockroot.save() - omitsfileroot = MakeOmitFileRoot(UNSEENS) + omitsfileroot = MakeFileRoot(UNSEENS) survexomitsroot = SurvexBlock( name=OMITBLOCK, survexpath="", survexfile=omitsfileroot, legsall=0, legslength=0.0 )