split out users from people

parsers/imports.py

@@ -9,6 +9,7 @@ import troggle.parsers.logbooks
 import troggle.parsers.people
 import troggle.parsers.QMs
 import troggle.parsers.scans
+import troggle.parsers.users
 import troggle.settings
 
 """Master data import.
@@ -30,7 +31,7 @@ def import_users():
     print("-- Importing troggle Users (users.json) to ", end="")
     print(django.db.connections.databases["default"]["NAME"])
     with transaction.atomic():
-        troggle.parsers.people.load_users()
+        troggle.parsers.users.load_users()
 
 def import_surveyscans():
     print("-- Importing Survey Scans and Wallets")

parsers/people.py

@@ -1,9 +1,6 @@
-import base64
 import csv
-import json
 import os
 import re
-from cryptography.fernet import Fernet
 from html import unescape
 from pathlib import Path
 
@@ -89,96 +86,7 @@ def troggle_slugify(longname):
         slug = f"{slug}_{slug_cache[slug]}"
     slug_cache[slug] = 1
     
-    return slug
+    return slug      
-    
-USERS_FILE = "users_e.json"
-ENCRYPTED_DIR = "encrypted"
-def load_users():
-    """These are the previously registered users of the troggle system.
-    """
-    PARSER_USERS = "_users"
-    DataIssue.objects.filter(parser=PARSER_USERS).delete()
-
-    key = settings.LONGTERM_SECRET_KEY # Django generated
-    k = base64.urlsafe_b64encode(key.encode("utf8")[:32]) # make Fernet compatible  
-    f = Fernet(k)
-    print(f)
-     
-    
-    jsonfile = settings.EXPOWEB / ENCRYPTED_DIR / USERS_FILE
-    jsonurl = "/" + str(Path(ENCRYPTED_DIR) / USERS_FILE)
-    if not (jsonfile.is_file()):
-        message = f"   ! Users json file does not exist: '{jsonfile}'"
-        DataIssue.objects.create(parser=PARSER_USERS, message=message)
-        print(message)
-        return None
-         
-    with open(jsonfile, 'r', encoding='utf-8') as json_f:
-        try:
-            registered_users_dict = json.load(json_f)
-        except FileNotFoundError:
-            print("File not found!")
-        except json.JSONDecodeError:
-            print("Invalid JSON format! - JSONDecodeError")
-        except Exception as e:
-            print(f"An exception occurred: {str(e)}")            
-            message = f"! Troggle USERs.  Failed to load {jsonfile} JSON file"
-            print(message)
-            DataIssue.objects.update_or_create(parser=PARSER_USERS, message=message, url=jsonurl)
-            return None
-    users_list = registered_users_dict["registered_users"]
-    
-    print(f" - {len(users_list)} users read from JSON")
-    for userdata in users_list:   
-        if userdata["username"]:
-            if userdata["username"] == "expo":
-                continue
-            if userdata["username"] == "expoadmin":
-                continue
-            try:
-                u = userdata["username"]
-                e_email = userdata["email"]
-                email = f.decrypt(e_email).decode()
-                print(f" - user: '{u} <{email}>' ")
-                if existing_user := User.objects.filter(username=userdata["username"]): # WALRUS
-                    # print(f" - deleting existing user '{existing_user[0]}' before importing")
-                    existing_user[0].delete()
-                user = User.objects.create_user(userdata["username"], email, "secret")
-                user.set_password = "secret" # stores hash not password
-                user.is_staff = False
-                user.is_superuser = False
-                user.save()
-            except Exception as e:
-                print(f"Exception <{e}>\nusers  in db: {len(User.objects.all())}\n{User.objects.all()}")
-                formatted_json = json.dumps(userdata, indent=4)
-                print(formatted_json)
-                return None
-        else:
-            print(f" - user: BAD username for {userdata} ")
-         # if userdata["date"] != "" or userdata["date"] != "None":
-            # message = f"! {str(self.walletname)} Date format not ISO {userdata['date']}. Failed to load from {jsonfile} JSON file"
-            # from troggle.core.models.troggle import DataIssue
-            # DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
-
-
-    ru = []
-    for u in User.objects.all():
-        if u.username == "expo":
-            continue
-        if u.username == "expoadmin":
-            continue
-        e_email = f.encrypt(u.email.encode("utf8")).decode()
-        ru.append({"username":u.username, "email": e_email, "password": u.password})
-        print(u.username, e_email)
-        original = f.decrypt(e_email).decode()
-        print(u.username, original)
-        
-    jsondict = { "registered_users": ru }
-    encryptedfile = settings.EXPOWEB / ENCRYPTED_DIR / "encrypt.json"
-    # with open(encryptedfile, 'w', encoding='utf-8') as json_f:
-        # json.dump(jsondict, json_f, indent=1)
-    # return True
-       
 
 def load_people_expos():
     """This is where the folk.csv file is parsed to read people's names.

parsers/users.py

@@ -0,0 +1,107 @@
+import base64
+import json
+import os
+from cryptography.fernet import Fernet
+from pathlib import Path
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.db import models
+
+from troggle.core.models.troggle import DataIssue, Expedition, Person, PersonExpedition
+
+"""These functions do not match how the stand-alone folk script works. So the script produces an HTML file which has 
+href links to pages in troggle which troggle does not think are right.
+The standalone script needs to be renedred defucnt, and all the parsing needs to be in troggle. Either that,
+or they should use the same code by importing a module.
+"""
+
+
+USERS_FILE = "users_e.json"
+ENCRYPTED_DIR = "encrypted"
+def load_users():
+    """These are the previously registered users of the troggle system.
+    """
+    PARSER_USERS = "_users"
+    DataIssue.objects.filter(parser=PARSER_USERS).delete()
+
+    key = settings.LONGTERM_SECRET_KEY # Django generated
+    k = base64.urlsafe_b64encode(key.encode("utf8")[:32]) # make Fernet compatible  
+    f = Fernet(k)
+    print(f)
+     
+    
+    jsonfile = settings.EXPOWEB / ENCRYPTED_DIR / USERS_FILE
+    jsonurl = "/" + str(Path(ENCRYPTED_DIR) / USERS_FILE)
+    if not (jsonfile.is_file()):
+        message = f"   ! Users json file does not exist: '{jsonfile}'"
+        DataIssue.objects.create(parser=PARSER_USERS, message=message)
+        print(message)
+        return None
+         
+    with open(jsonfile, 'r', encoding='utf-8') as json_f:
+        try:
+            registered_users_dict = json.load(json_f)
+        except FileNotFoundError:
+            print("File not found!")
+        except json.JSONDecodeError:
+            print("Invalid JSON format! - JSONDecodeError")
+        except Exception as e:
+            print(f"An exception occurred: {str(e)}")            
+            message = f"! Troggle USERs.  Failed to load {jsonfile} JSON file"
+            print(message)
+            DataIssue.objects.update_or_create(parser=PARSER_USERS, message=message, url=jsonurl)
+            return None
+    users_list = registered_users_dict["registered_users"]
+    
+    print(f" - {len(users_list)} users read from JSON")
+    for userdata in users_list:   
+        if userdata["username"]:
+            if userdata["username"] == "expo":
+                continue
+            if userdata["username"] == "expoadmin":
+                continue
+            try:
+                u = userdata["username"]
+                e_email = userdata["email"]
+                email = f.decrypt(e_email).decode()
+                print(f" - user: '{u} <{email}>' ")
+                if existing_user := User.objects.filter(username=userdata["username"]): # WALRUS
+                    # print(f" - deleting existing user '{existing_user[0]}' before importing")
+                    existing_user[0].delete()
+                user = User.objects.create_user(userdata["username"], email, "secret")
+                user.set_password = "secret" # stores hash not password
+                user.is_staff = False
+                user.is_superuser = False
+                user.save()
+            except Exception as e:
+                print(f"Exception <{e}>\nusers  in db: {len(User.objects.all())}\n{User.objects.all()}")
+                formatted_json = json.dumps(userdata, indent=4)
+                print(formatted_json)
+                return None
+        else:
+            print(f" - user: BAD username for {userdata} ")
+         # if userdata["date"] != "" or userdata["date"] != "None":
+            # message = f"! {str(self.walletname)} Date format not ISO {userdata['date']}. Failed to load from {jsonfile} JSON file"
+            # from troggle.core.models.troggle import DataIssue
+            # DataIssue.objects.update_or_create(parser="wallets", message=message, url=wurl)
+
+
+    ru = []
+    for u in User.objects.all():
+        if u.username == "expo":
+            continue
+        if u.username == "expoadmin":
+            continue
+        e_email = f.encrypt(u.email.encode("utf8")).decode()
+        ru.append({"username":u.username, "email": e_email, "password": u.password})
+        print(u.username, e_email)
+        original = f.decrypt(e_email).decode()
+        print(u.username, original)
+        
+    jsondict = { "registered_users": ru }
+    encryptedfile = settings.EXPOWEB / ENCRYPTED_DIR / "encrypt.json"
+    # with open(encryptedfile, 'w', encoding='utf-8') as json_f:
+        # json.dump(jsondict, json_f, indent=1)
+    # return True
+