Makefile docm

handbook/troggle/scriptscurrent.html

@@ -29,8 +29,12 @@ p {
 
 <h4 id="makefile">Makefiles</h4>
 <p>In :expoweb:/noinfo/
-<p>This may be obsolete. It used to coordinate running essential updates but also definitely includes redundant  stuff. It <span style="color:red">needs some serious attention</span>.
+<p>This may be obsolete. It used to coordinate running essential updates but also definitely includes redundant  stuff. It <span 
-<p>It coordinates producing the 3d surveys used in the cave description pages, updates the area pages, runs the folk script, runs the QM list generation within each of the cave pages that needs it, runs svxtrace, and reports on everything using "bigbro" which we don't have any other reference to. (Generation of the .3d files as required is now done by troggle.)
+style="color:red">needs some serious attention</span>.
+<p>It coordinates producing the 3d surveys used in the cave description pages, updates the area pages, runs the folk script, 
+runs the QM list generation within each of the cave pages that needs it, runs <a href="#svxtrace">svxtrace</a>, 
+and reports on everything using "bigbro" which we don't have any other reference to. 
+<p>Today, troggle generates the .3d and .pos files, parses and loads the QM list and parses the include tree of the survex files.
 
 <h4 id="wallets">Wallets</h4>
 <p><a href="../survey/onlinewallet.html">Online wallets</a> are initially maintained using the <a href="/expofiles/surveyscans/wallets.py">wallets.py</a> script, but troggle also directly imports all the expofiles/surveyscans/ directories of scanned survey notes and produces <a href="/survey_scans/">reports</a> on them. There are several bash and python scripts in the  <a href="/expofiles/surveyscans/">surveyscans</a> directory to create wallets for the coming year, and to re-run the wallet processing on all  past years (for when we improve the script). For 2021 we have converted wallets.py to python3, so be careful of older versions which are python2.
@@ -82,7 +86,10 @@ See the entire page devoted to the various
 <a href="scriptsqms.html">QM scripts</a>.
 
 <h4 id="svxtrace">svxtrace.py</h4>
-<p>Traces all the svx file dependencies via the *include statements. In :expoweb:/ . The documented workflow today does not appear to need this, but that might be a documentation fault. It might do a key job. <span style="color:red">[to be investigated]</span>
+<p>Traces all the svx file dependencies via the *include statements. 
+In :expoweb:/ . The documented workflow today does not need this, but that might be a documentation fault. 
+It might do a key job. It is used by <a href="#makefile">makefile</a> above. 
+Its job has almost certainly been superceded by the survex file parser in troggle.
 
 <h4 id="survex">Survex files</a> - reference checking</h4>
 <p>Survex files contain a reference to the wallet which contains the original survey notes for that surveyed passage. These sometimes have errors and also get out of date as caves get renamed when they get a kataster number issued. Each online survey wallet also has a reference to the survex file(s) which have been typed up from that data.

noinfo/scripts/Makefile

@@ -0,0 +1,155 @@
+# Makefile to run the various scripts needed once hg and git updates have been performed.
+
+# This desperately needs properly revising by someone who knows both the history and also how things are done now [April 2021]
+
+dataset_path = ../loser
+surveydir_path = ../surveys
+cavern_flags = -qq -s -v7
+tar_flags = -cz --exclude "*/CVS" --ignore-case -f
+svxtrace_cmd = ./svxtrace.py
+
+# 3dtopos = ?
+
+#things needed to run this stuff
+#python3 imaging library, freesans or arial truetype font, dump3d, cavestats
+#Debian packages: python3-imaging, ttf-freefont, perl, python3
+#dump3d needs to be built in configured survex sources with cd src;make dump3d
+#
+
+###########
+
+#everything: autogen_pages folklist 3dfiles tarfiles surveytable qmlists prospecting_guide
+everything: autogen_pages folklist 3dfiles tarfiles qmlists
+
+###########
+
+autogen_pages: noinfo/lengths.dat noinfo/1623-and-1626.pos noinfo/cavestats
+
+# cavestats should be BUILT AUTOMATICALLY rather than having to be
+# built manually in a survex build tree.
+#indxal.htm: noinfo/CAVETAB2.CSV noinfo/all.pos noinfo/make-indxal4.pl noinfo/areapage_skeletons/*.html noinfo/lengths.dat
+#	python3 noinfo/make-areaindices.py noinfo/cavestats
+#	cd noinfo; perl ./make-indxal4.pl -q CAVETAB2.CSV
+
+noinfo/lengths.dat:
+	python3 noinfo/create_dplong_table.py
+
+noinfo/1623-and-1626.pos: noinfo/1623-and-1626.3d
+	3dtopos noinfo/1623-and-1626.3d noinfo/1623-and-1626.pos
+
+noinfo/cavestats:
+	# This needs a built copy of survex sources. See
+	# noinfo/cavestats.build for details
+
+###########
+
+folklist: folk/index.htm $(dataset_path)/1623-and-1626.svx
+
+folk/index.htm: folk/folk.csv scripts/make-folklist.py
+	python3 scripts/make-folklist.py < folk/folk.csv > folk/index.htm
+
+###########
+
+#This is now part of separate repo and should be built separately 
+#surveytable:
+#	cd $(surveydir_path); perl tablize-csv.pl Surveys.csv; perl tablizebyname-csv.pl Surveys.csv --quiet
+
+
+
+# Output depends on entire contents of the survey tree;
+# there appears to be no sensible way to run this at need.
+
+###########
+#QM files are now outo built in troggle for all caves-1623/
+
+  qmlists: 1623/204/atoz.html 
+
+# qmlists: 1623/204/qm.html 1623/204/atoz.html 1623/234/qm.html 1623/258/qm.html
+
+# 1623/204/qm.html: 1623/204/qm.csv 1623/204/tablize-qms.pl
+	# cd 1623/204; perl tablize-qms.pl qm.csv
+
+# 1623/234/qm.html: 1623/234/qm.csv 1623/234/tablize-qms.pl
+	# cd 1623/234; perl tablize-qms.pl qm.csv
+
+# 1623/258/qm.html: 1623/258/qm.csv 1623/258/tablize-qms.pl
+	# cd 1623/258; perl tablize-qms.pl qm.csv
+
+1623/204/atoz.html: 1623/204/glossary.csv 1623/204/make-glossary.pl
+	cd 1623/204; perl make-glossary.pl glossary.csv
+
+###########
+#3d files are now outo built in troggle for all caves-1623/
+  3dfiles: noinfo/1623-and-1626.3d noinfo/alltracks.3d
+
+# 3dfiles: noinfo/1623-and-1626.3d 1623/204/204.3d 1623/234/234.3d 1623/76/76.3d noinfo/alltracks.3d
+
+# noinfo/1623-and-1626.3d: $(shell $(svxtrace_cmd) $(dataset_path)/1623-and-1626)
+	# cavern $(cavern_flags) $(dataset_path)/1623-and-1626 -o noinfo/1623-and-1626.3d
+
+# 1623/204/204.3d: $(shell $(svxtrace_cmd) $(dataset_path)/caves-1623/204/204)
+	# cavern $(cavern_flags) $(dataset_path)/caves-1623/204/204 -o 1623/204/204.3d
+
+# 1623/234/234.3d: $(shell $(svxtrace_cmd) $(dataset_path)/caves-1623/234/234)
+	# cavern $(cavern_flags) $(dataset_path)/caves-1623/234/234 -o 1623/234/234.3d
+
+# 1623/76/76.3d: $(shell $(svxtrace_cmd) $(dataset_path)/caves-1623/76/76)
+	# cavern $(cavern_flags) $(dataset_path)/caves-1623/76/76 -o 1623/76/76.3d
+
+# noinfo/alltracks.3d: $(shell $(svxtrace_cmd) $(dataset_path)/fixedpts/gps/tracks/alltracks)
+	# cavern $(cavern_flags) $(dataset_path)/fixedpts/gps/tracks/alltracks.svx -o noinfo/alltracks.3d
+
+###########
+
+#this requires ttf-freefont package to be installed
+#And dump3d from survex
+prospecting_guide: 3dfiles freefont noinfo/lengths.dat
+	cd noinfo/prospecting_guide_scripts && python3 make-prospectingguide-new.py
+	cd noinfo/prospecting_guide_scripts && python3 make-prospectingguide-new.py --white
+
+freefont: /usr/share/fonts/truetype/freefont/FreeSans.ttf
+        # ttf-freefont needs to be installed (or MS corefonts) 
+
+
+dump3d: noinfo/prospecting_guide_scripts/dump3d
+       # apt-get source survex && sudo apt-get build-dep survex
+       # cd survex-* && debuild
+       # make dump3d && cp dump3d $(pwd)/noinfo/prospecting_guide_scripts/dump3d
+
+###########
+# We don't do the tarfile thing anymore
+  tarfiles: noinfo/all.tgz 
+
+# tarfiles: noinfo/all.tgz 1623/204/surveydata.tgz 1623/234/surveydata.tgz 1623/76/surveydata.tgz
+
+  noinfo/all.tgz: noinfo/1623-and-1626.3d
+	tar $(tar_flags) noinfo/all.tgz -C $(dataset_path)/.. loser
+
+# 1623/204/surveydata.tgz: 1623/204/204.3d
+	# tar $(tar_flags) 1623/204/surveydata.tgz -C $(dataset_path)/caves-1623/ 204
+
+# 1623/234/surveydata.tgz: 1623/234/234.3d
+	# tar $(tar_flags) 1623/234/surveydata.tgz -C $(dataset_path)/caves-1623/ 234
+
+# 1623/76/surveydata.tgz: 1623/76/76.3d
+	# tar $(tar_flags) 1623/76/surveydata.tgz -C $(dataset_path)/caves-1623/ 76
+
+# (tarfiles depend on 3d files to avoid running the
+# expensive $(shell ...) commands more than necessary.)
+
+###########
+
+clean:
+	#cut -d "," -f 6 noinfo/CAVETAB2.CSV | xargs rm -f
+	rm -f indxal.htm folk/index.htm noinfo/all.pos
+	rm -f noinfo/1623-and-1626.3d noinfo/1623.3d 1623/204/204.3d 1623/234/234.3d 1623/76/76.3d
+	rm -f 1623/204/surveydata.tgz 1623/234/surveydata.tgz 1623/76/surveydata.tgz noinfo/all.tgz
+	rm -f $(surveydir_path)/surveytable.html
+	rm -f report.html
+	find . -name ".cvsignore" | xargs rm -f
+
+###########
+
+report: everything
+	rm -f report.html
+	bigbro -ohtml report.html -local -fragments -failures -rec .* -ignore ".*/jnl/.*" -ignore ".*/icons/.*" -ignore ".*rdmap.*" -ignore ".*/cp/.*" -ignore ".*/surveys/.*" index.htm