[svn] Work on turn html pages into cavedescription models.py.

Moved parser/cavetabs html_to_wiki function to utils.py
Added databaseReset.py desc to refresh the cavedescriptions.

utils.py

@@ -1,5 +1,8 @@
 from django.conf import settings
-from troggle.core.models import LogbookEntry
+try:
+    from django.db import models
+except:#We want to get rid of this try statement if possible
+    from troggle.core.models import LogbookEntry
 import random, re, logging
 
 def weighted_choice(lst):
@@ -16,11 +19,11 @@ def randomLogbookSentence():
     # needs to handle empty logbooks without crashing
 
     #Choose a random logbook entry
-    randSent['entry']=LogbookEntry.objects.order_by('?')[0]
+    randSent['entry']=models.LogbookEntry.objects.order_by('?')[0]
 
     #Choose again if there are no sentances (this happens if it is a placeholder entry)
     while len(re.findall('[A-Z].*?\.',randSent['entry'].text))==0:
-        randSent['entry']=LogbookEntry.objects.order_by('?')[0]
+        randSent['entry']=models.LogbookEntry.objects.order_by('?')[0]
     
     #Choose a random sentence from that entry. Store the sentence as randSent['sentence'], and the number of that sentence in the entry as randSent['number']
     sentenceList=re.findall('[A-Z].*?\.',randSent['entry'].text)
@@ -60,10 +63,98 @@ def save_carefully(objectType, lookupAttribs={}, nonLookupAttribs={}):
 
 def render_with_context(req, *args, **kwargs):
     """this is the snippet from http://www.djangosnippets.org/snippets/3/
-    
-    Django uses Context, not RequestContext when you call render_to_response. We always want to use RequestContext, so that django adds the context from settings.TEMPLATE_CONTEXT_PROCESSORS. This way we automatically get necessary settings variables passed to each template. So we use a custom method, render_response instead of render_to_response. Hopefully future Django releases will make this unnecessary."""
+
+    Django uses Context, not RequestContext when you call render_to_response.
+    We always want to use RequestContext, so that django adds the context from
+    settings.TEMPLATE_CONTEXT_PROCESSORS. This way we automatically get
+    necessary settings variables passed to each template. So we use a custom
+    method, render_response instead of render_to_response. Hopefully future
+    Django releases will make this unnecessary."""
 
     from django.shortcuts import render_to_response
     from django.template import RequestContext
     kwargs['context_instance'] = RequestContext(req)
-    return render_to_response(*args, **kwargs)
+    return render_to_response(*args, **kwargs)
+    
+re_body = re.compile(r"\<body[^>]*\>(.*)\</body\>", re.DOTALL)
+re_title = re.compile(r"\<title[^>]*\>(.*)\</title\>", re.DOTALL)
+def get_html_body(text):
+    return get_single_match(re_body, text)
+
+def get_html_title(text):
+    return get_single_match(re_title, text)
+
+def get_single_match(regex, text):
+    match = regex.search(text)
+
+    if match:
+        return match.groups()[0]
+    else:
+        return None
+
+
+re_subs = [(re.compile(r"\<b[^>]*\>(.*?)\</b\>", re.DOTALL), r"'''\1'''"),
+           (re.compile(r"\<i\>(.*?)\</i\>", re.DOTALL), r"''\1''"),
+           (re.compile(r"\<h1[^>]*\>(.*?)\</h1\>", re.DOTALL), r"=\1="),
+           (re.compile(r"\<h2[^>]*\>(.*?)\</h2\>", re.DOTALL), r"==\1=="),
+           (re.compile(r"\<h3[^>]*\>(.*?)\</h3\>", re.DOTALL), r"===\1==="),
+           (re.compile(r"\<h4[^>]*\>(.*?)\</h4\>", re.DOTALL), r"====\1===="),
+           (re.compile(r"\<h5[^>]*\>(.*?)\</h5\>", re.DOTALL), r"=====\1====="),
+           (re.compile(r"\<h6[^>]*\>(.*?)\</h6\>", re.DOTALL), r"======\1======"),
+           (re.compile(r"\<a\s+id=['\"]([^'\"]*)['\"]\s*\>(.*?)\</a\>", re.DOTALL), r"[subcave:\1|\2]"),
+           #interpage link needed
+           (re.compile(r"\<a\s+href=['\"]#([^'\"]*)['\"]\s*\>(.*?)\</a\>", re.DOTALL), r"[cavedescription:\1|\2]"),
+           (re.compile(r"\[\<a\s+href=['\"][^'\"]*['\"]\s+id=['\"][^'\"]*['\"]\s*\>([^\s]*).*?\</a\>\]", re.DOTALL), r"![qm:\1]"),
+
+           ]
+
+def html_to_wiki(text, codec = "utf-8"):
+    if type(text) == str:
+        text = unicode(text, codec)
+    text = re.sub("</p>", r"", text)
+    text = re.sub("<p>$", r"", text)
+    text = re.sub("<p>", r"\n\n", text)
+    out = ""
+    lists = ""
+    #lists
+    while text:
+        mstar = re.match("^(.*?)<ul[^>]*>\s*<li[^>]*>(.*?)</li>(.*)$", text, re.DOTALL)
+        munstar = re.match("^(\s*)</ul>(.*)$", text, re.DOTALL)
+        mhash = re.match("^(.*?)<ol[^>]*>\s*<li[^>]*>(.*?)</li>(.*)$", text, re.DOTALL)
+        munhash = re.match("^(\s*)</ol>(.*)$", text, re.DOTALL)
+        mitem = re.match("^(\s*)<li[^>]*>(.*?)</li>(.*)$", text, re.DOTALL)
+        ms = [len(m.groups()[0]) for m in [mstar, munstar, mhash, munhash, mitem] if m]
+        def min_(i, l):
+            try:
+                v = i.groups()[0]
+                l.remove(len(v))
+                return len(v) < min(l, 1000000000)
+            except:
+                return False
+        if min_(mstar, ms):
+            lists += "*"
+            pre, val, post = mstar.groups()
+            out += pre + "\n" + lists + " " + val
+            text = post
+        elif min_(mhash, ms):
+            lists += "#"
+            pre, val, post = mhash.groups()
+            out += pre + "\n" + lists + " " + val
+            text = post
+        elif min_(mitem, ms):
+            pre, val, post = mitem.groups()
+            out += "\n" + lists + " " + val
+            text = post
+        elif min_(munstar, ms):
+            lists = lists[:-1]
+            text = munstar.groups()[1]
+        elif min_(munhash, ms):
+            lists.pop()
+            text = munhash.groups()[1]
+        else:
+            out += text
+            text = ""
+    #substitutions
+    for regex, repl in re_subs:
+        out = regex.sub(repl, out)
+    return out