troggle/core/TESTS/tests.py

"""
We are using unittest for troggle.

Note that the database has not been parsed from the source files when these tests are run,
so any path that relies on data being in the database will fail.

The simple redirections to files which exist, e.g. in
/expoweb/
/expofiles/
/expofiles/documents/
etc. will test fine.

But paths like this:
/survey_scans/
/caves/
which rely on database resolution will fail unless a fixture has been set up for
them.

https://docs.djangoproject.com/en/3.0/topics/testing/tools/
"""
import unittest
import re
from django.test import TestCase, SimpleTestCase, Client

class SimpleTest(SimpleTestCase):
    def test_test_setting(self):
        from django.conf import settings        
        self.assertEqual(settings.EMAIL_BACKEND, 'django.core.mail.backends.locmem.EmailBackend')
    def test_import_TroggleModel(self):
        from troggle.core.models import TroggleModel
    def test_import_Cave(self):
        from troggle.core.models_caves import Cave
    def test_import_parsers_surveys(self):
        from PIL import Image
        from utils import save_carefully
        from functools import reduce
    def test_import_parsers_survex(self):
        import troggle.settings as settings
        import troggle.core.models as models
        import troggle.core.models_caves as models_caves
        import troggle.core.models_survex as models_survex
        from troggle.parsers.people import GetPersonExpeditionNameLookup
        from troggle.core.views.caves import MapLocations
    def test_import_parsers_QMs(self):
        from troggle.core.models_caves import QM, Cave, LogbookEntry
        from utils import save_carefully
    def test_import_parsers_people(self):
        from html.parser import HTMLParser
        from unidecode import unidecode
    def test_import_parsers_logbooks(self):
        from django.template.defaultfilters import slugify
        from django.utils.timezone import get_current_timezone, make_aware
        from troggle.core.models import DataIssue, Expedition
        from troggle.core.models_caves import Cave, LogbookEntry, PersonTrip
        from parsers.people import GetPersonExpeditionNameLookup
    def test_import_core_views_caves(self):
        from django.http import HttpResponse, HttpResponseRedirect
        from django.shortcuts import get_object_or_404, render
        import troggle.core.views.expo 
        from troggle.core.models import Expedition
        from troggle.core.models_caves import CaveSlug, Cave, CaveAndEntrance, QM, EntranceSlug, Entrance, Area, SurvexStation
        from troggle.core.forms import CaveForm, CaveAndEntranceFormSet, VersionControlCommentForm, EntranceForm, EntranceLetterForm
        from troggle.core.views.login import login_required_if_public
        from django.contrib.auth.decorators import login_required
        from django.conf import settings
    def test_import_parsers_mix(self):
        from troggle.parsers.logbooks import GetCaveLookup
        import troggle.settings
        import troggle.logbooksdump
        import troggle.parsers.caves
        import troggle.parsers.people
        import troggle.parsers.surveys
        import troggle.parsers.logbooks
        import troggle.parsers.QMs
        import troggle.parsers.survex 
    def test_import_imports(self):
        from django.core import management
        from django.db import connection, close_old_connections, connections
        from django.contrib.auth.models import User
        from django.http import HttpResponse
        from django.urls import reverse

    def test_import_urls(self):
        from django.conf import settings
        from django.conf.urls import url, include
        from django.views.generic.base import RedirectView 
        from django.views.generic.edit import UpdateView
        from django.views.generic.list import ListView
        from django.contrib import admin
        from django.urls import reverse, resolve


#class SimplePageTest(unittest.TestCase):
class PageTests(TestCase):
    '''These tests may appear to be redundant, but in fact they exercise different bits of code. The urls.py
    dispatcher is sending these URLs view via different 'view' handlers, and they all need verifying.
    '''
    @classmethod
    def setUpTestData(cls):
        # Set up data for the whole TestCase
        #cls.foo = Foo.objects.create(bar="Test")
           # Some test using self.foo in tests below..
        # read in some SQL ?
        pass
        
    def setUp(self):
        # Every test needs a client.
        self.client = Client()

    def test_expoweb_root(self):
        response = self.client.get('')
        content = response.content.decode()
        self.assertEqual(response.status_code, 200)
        ph = r'CUCC in Austria'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_expoweb_root_slash(self):
        response = self.client.get('/')
        content = response.content.decode()
        self.assertEqual(response.status_code, 200)
        ph = r'CUCC in Austria'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_expoweb_dir(self):
        response = self.client.get('/handbook')
        content = response.content.decode()
        self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm

    def test_expoweb_dirslash(self):
        response = self.client.get('/handbook/')
        content = response.content.decode()
        self.assertEqual(response.status_code, 302) # directory, so redirects to /index.htm
        
    def test_expoweb_dir_no_index(self):
        response = self.client.get('/handbook/troggle')
        content = response.content.decode()
        self.assertEqual(response.status_code, 200)
        ph = r'Page not found handbook/troggle/index.html'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")
  
    def test_expoweb_dir_with_index_htm(self):
        response = self.client.get('/years/1999/index.htm')
        content = response.content.decode()
        self.assertEqual(response.status_code, 200) # directory, so redirects to /index.htm
        ph = r'Passage descriptions for 1999'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")
    
    def test_expoweb_dir_with_index_html(self):
        response = self.client.get('/years/2015/index.html')
        content = response.content.decode()
        self.assertEqual(response.status_code, 200) # directory, so redirects to /index.htm
        ph = r'Things left at top camp 2014'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_expoweb_dir_with_index2(self):
        response = self.client.get('/handbook/index.htm')
        content = response.content.decode()
        self.assertEqual(response.status_code, 200) 
        ph = r'Introduction to expo'
        phmatch    = re.search(ph, content)
        #print("\n ! - test_expoweb_dir_with_index2\n{}\n{}".format(response.reason_phrase, content))
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")
 
    def test_expoweb_htm(self):
        response = self.client.get('/handbook/index.htm')
        content = response.content.decode()
        self.assertEqual(response.status_code, 200)
        ph = r'Introduction to expo'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_expoweb_notfound(self):
        response = self.client.get('/handbook/zyxxypqrqx.html')
        content = response.content.decode()
        self.assertEqual(response.status_code, 200)
        ph = r'<h1>Page not found'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_expoweb_no_dir(self):
        # slash where there should not be one
        response = self.client.get('/handbook/zyxxypqrqx/')
        self.assertEqual(response.status_code, 200)
        content = response.content.decode()
        ph = r"<h1>Directory not found"
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_expoweb_via_areaid(self):
        # the dispatcher takes a detour via the cave renering procedure for this
        response = self.client.get('/1623/others/t/via201.jpg')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(len(response.content), 6057) 

    def test_cave_kataster_not_found(self):
        # database not loaded, so no caves found
        response = self.client.get('/cave/115')
        self.assertEqual(response.status_code, 200)
        content = response.content.decode()
        ph = r"Cave not found in database"
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")



    def test_page_admin(self):
        # see the login page
        response = self.client.get('/admin/login/')
        content = response.content.decode()
        self.assertEqual(response.status_code, 200)
        ph = r'<h1 id="site-name">Troggle administration</h1>'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_admindocs_exped(self):
        # Get redirected to login page
        response = self.client.get('/admin/doc/models/core.expedition/')
        content = response.content.decode()
        self.assertEqual(response.status_code, 302)

    def test_page_expofiles_dir(self):
        # Flat file tests.
        response = self.client.get('/expofiles/')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            content = response.content.decode()       
            for ph in [ r'a href="/expofiles/geotiffsurveys">/geotiffsurveys/',
                    r'<a href="/expofiles/photos">/photos/',
                    r'<a href="/expofiles/surveyscans">/surveyscans' ]:
                phmatch    = re.search(ph, content)
                self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_survey_scans_dir(self):
        # Flat file tests.
        response = self.client.get('/expofiles/surveyscans')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            content = response.content.decode()       
            for ph in [ r'<a href="/expofiles/surveyscans/2004">/2004/',
                    r'<a href="/expofiles/surveyscans/1989LUSS">/1989LUSS/',
                    r'<a href="/expofiles/surveyscans/2018">/2018' ]:
                phmatch    = re.search(ph, content)
                self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_folk(self):
        # This page is separately generated, so it has the full data content
        response = self.client.get('/folk/index.htm')
        content = response.content.decode()       
        self.assertEqual(response.status_code, 200) 
        for ph in [ r'involves some active contribution',
                r'Naomi Griffiths',
                r'Gail Smith',
                r'Phil Wigglesworth',
                r'A more obscure record of longest gap between expos has' ]:
            phmatch    = re.search(ph, content)
            self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_expofile_documents(self):
        # this gets an empty page as the database has not been loaded
        response = self.client.get('/expofiles/documents')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            content = response.content.decode()
            ph = r'notice_generale_cordes_courant'
            phmatch    = re.search(ph, content)
            self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_expofile_documents_slash(self):
        # this gets an empty page as the database has not been loaded
        response = self.client.get('/expofiles/documents/')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            content = response.content.decode()
            ph = r'notice_generale_cordes_courant'
            phmatch    = re.search(ph, content)
            self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")



    def test_page_expofile_document_loeffler_pdf(self):
        # Flat file tests.
        response = self.client.get('/expofiles/documents/surveying/tunnel-loefflerCP35-only.pdf')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            self.assertEqual(len(response.content), 2299270) 

    def test_page_expofile_document_rope_pdf(self):
        # Flat file tests.
        response = self.client.get('/expofiles/documents/rope-age-agm-2019.pdf')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            self.assertEqual(len(response.content), 76197) 

    def test_page_expofile_document_png(self):
        # Flat file tests.
        response = self.client.get('/expofiles/documents/callout-2012.png')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            self.assertEqual(len(response.content), 69921) 

    def test_page_expofile_writeup(self):
        # Flat file tests.
        response = self.client.get('/expofiles/writeups/1982/logbook1982.pdf')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            self.assertEqual(len(response.content), 12915413) 
 
    def test_page_site_media_ok(self):
        # Flat file tests.
        response = self.client.get('/site_media/surveyHover.gif')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            self.assertEqual(len(response.content), 39482 ) # need to check it is not just an error page
 
    def test_page_site_media_css(self):
        # Flat file tests.
        response = self.client.get('/site_media/css/main3.css')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            content = response.content.decode() # need to check it is not just an error page
            ph = r'This text is used by the test system to determine that main3.css loaded correctly'
            phmatch    = re.search(ph, content)
            self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_photos_ok(self):
        # Flat file tests.
        response = self.client.get('/photos/2018/PhilipSargent/corin.jpg')
        if response.status_code != 200:
            self.assertEqual(response.status_code, 302)  
        if response.status_code != 302:
            self.assertEqual(response.status_code, 200)  
            self.assertEqual(len(response.content), 67487 ) # need to check it is not just an error page

    def test_page_photos_not_ok(self):
        # Flat file tests.
        response = self.client.get('/photos/2018/PhilipSargent/corin.jpeg')
        self.assertEqual(response.status_code, 200)
        content = response.content.decode()
        ph = r'<title>Page not found 2018/PhilipSargent/corin.jpeg</title>'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_photos_dir(self):
        # Flat file tests.
        response = self.client.get('/photos/2018/PhilipSargent/')
        self.assertEqual(response.status_code, 200)
        content = response.content.decode()
        ph = r'Directory not displayed'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_survey_scans_empty(self):
        # this gets an empty page as the database has not been loaded
        response = self.client.get('/survey_scans/')
        self.assertEqual(response.status_code, 200)
        content = response.content.decode()
        ph = r'contains the scanned original in-cave survey notes and sketches'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_tunneldataraw_empty(self):
        # this gets an empty page as the database has not been loaded
        response = self.client.get('/tunneldataraw/')
        self.assertEqual(response.status_code, 200)
        content = response.content.decode()
        ph = r"<h1>Directory not found"
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_page_slash_empty(self):
        # tslash where there should not be one
        response = self.client.get('/expedition/1979/')
        self.assertEqual(response.status_code, 200)
        content = response.content.decode()
        ph = r"<h1>Directory not found"
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

    def test_not_found_survexfile_cave(self):
        response = self.client.get('/survexfile/not_a_real_cave_number')
        self.assertEqual(response.status_code, 200)
        content = response.content.decode()
        ph = r'Cave not found in database'
        phmatch    = re.search(ph, content)
        self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")

# ADD TESTS when we are redirecting /expofiles/ to get the actual files using e.g.
# import requests
# page = requests.get("http://dataquestio.github.io/web-scraping-pages/simple.html")

# these need a fixture to load the datbase before they will pass
# we also need tests for invalid queries to check that error pages are right

    # def test_page_survey_scans_khplan2_png(self):
        # # this has an error as the database has not been loaded yet in the tests
        # response = self.client.get('/survey_scans/smkhs/khplan2.png')
        # if response.status_code != 200:
            # self.assertEqual(response.status_code, 302)  
        # if response.status_code != 302:
            # self.assertEqual(response.status_code, 200)  
        # self.assertEqual(len(response.content), 823304) # fails, but is working manually!

    # def test_page_tunneldataraw_107sketch_xml(self):
        # # this has an error as the database has not been loaded yet in the tests
        # response = self.client.get('/tunneldataraw/107/107sketch-v2.xml')
        # if response.status_code != 200:
            # self.assertEqual(response.status_code, 302)  
        # if response.status_code != 302:
            # self.assertEqual(response.status_code, 200)  
        # content = response.content.decode()       
        # for ph in [ r'tunneldate="2014-08-21 11:34:00"',
                # r'<sketchsubset subname="Caves of the Loser Plateau"/>',
                # r'sfsketch="ollyjen107drawings',
                # r'sfsketch="surveyscans/2014/2014#01',
                # r'aa-js-plan.png"' ]:
            # phmatch    = re.search(ph, content)
            # self.assertIsNotNone(phmatch, "Failed to find expected text: '" + ph +"'")


  
        # database not loaded yet:
        #response = self.client.get('/survey_scans/1991surveybook/page0002.png')
        #response = self.client.get('/survey_scans/1991surveybook/')
        #content = response.content.decode()
        #print(content)
        #png93    = re.search(r'/page0093.png">page0093.png</a></td>', content)