Switch: drone ---> gitea actions

- To make sure the newest version is returned.

.drone.yml

@@ -1,27 +0,0 @@
----
-kind: pipeline
-type: exec
-name: default
-platform:
-  os: linux
-  arch: amd64
-steps:
-- name: Run unit test
-  environment:
-   PYTHONPATH: .
-  commands:
-  - python setup.py test
-- name: Build package files
-  commands:
-  - python setup.py sdist
-- name: Versie toevoegen aan download
-  environment:
-    DOWNLOADDIR: /usr/share/nginx/html/download/python/ddoperapi
-  commands:
-  - mkdir -p "$${DOWNLOADDIR}"
-  - cd dist
-  - for f in *;do if [ -f "$${DOWNLOADDIR}/$${f}" ] ; then echo version error;exit 1;else cp "$${f}" "$${DOWNLOADDIR}";fi;done
-  - ls -l "$${DOWNLOADDIR}"
-  when:
-    branch:
-    - master

.gitea/workflows/build.yml

@@ -0,0 +1,91 @@
+---
+name: build
+on:
+  - push
+
+jobs:
+  testFedoraBuilder:
+    runs-on: fedora-builder
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Test
+        run: tox
+  testRockylinux8:
+    runs-on: rockylinux8
+    steps:
+      - name: add ca
+        run: |
+          curl -o /etc/pki/ca-trust/source/anchors/ca-marceln.org.crt \
+                  https://marceln.org/ca-marceln.org.crt
+          update-ca-trust
+      - name: Update yum conf
+        run: "/usr/bin/sed -i -e \
+              's@#baseurl=http://dl.rockylinux.org/$contentdir@\
+              baseurl=https://rock64.fritz.box/nluug/os/Linux/distr/rocky@' \
+              -e 's@^mirrorlist=https@#mirrorlist=https@' \
+              $(grep -l '#baseurl=http://dl.rockylinux.org/$contentdir' \
+              /etc/yum.repos.d/*)"
+      - name: Setup
+        run: |
+          dnf install -y git nodejs \
+                      python36 python38 python39 python3.11 python3.12 \
+                      python3.11-pip python3.12-pip
+          pip-3 install tox
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Test
+        run: tox -e py36,py38,py39,py311
+  testRockylinux9:
+    runs-on: rockylinux9
+    steps:
+      - name: add ca
+        run: |
+          curl -o /etc/pki/ca-trust/source/anchors/ca-marceln.org.crt \
+                  https://marceln.org/ca-marceln.org.crt
+          update-ca-trust
+      - name: Update yum conf
+        run: "/usr/bin/sed -i -e \
+              's@#baseurl=http://dl.rockylinux.org/$contentdir@\
+              baseurl=https://rock64.fritz.box/nluug/os/Linux/distr/rocky@' \
+              -e 's@^mirrorlist=https@#mirrorlist=https@' \
+              $(grep -l '#baseurl=http://dl.rockylinux.org/$contentdir' \
+              /etc/yum.repos.d/*)"
+      - name: Setup
+        run: |
+          dnf install -y git nodejs \
+                      python39 python3.11 python3.12 \
+                      python3-pip python3.11-pip python3.12-pip
+          pip-3 install tox
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Test
+        run: tox -e py39,py311,py312
+  testUbuntu22:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Setup
+        run: |
+          apt update
+          apt install -y git nodejs python3 python3-pip tox
+          pip
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Test
+        run: tox -e py310
+  buildAndPublish:
+    needs: [testFedoraBuilder, testRocky8, testRocky9, testUbuntu22]
+    runs-on: fedora-builder
+    env:
+      TWINE_USERNAME: $(( secrets.PYPI_USER ))
+      TWINE_PASSWORD: $(( secrets.PYPI_PASSWORD ))
+    if: gitea.ref_name == 'main'
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Build
+        run: python3 -m build
+      - name: Test
+        run: tox
+      - name: Upload naar pypi
+        run: python3 -m twine upload dist/*

.gitignore

@@ -2,3 +2,9 @@
 __pycache__
 dd_oper_api.egg-info
 dist
+dependency_links.txt
+PKG-INFO
+requires.txt
+SOURCES.txt
+top_level.txt
+.tox

LICENSE

@@ -1 +1,20 @@
-TODO
+Copyright (c) 2022 Rijkswaterstaat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

README.md

@@ -9,14 +9,14 @@ gebruikt voor het opvragen van:
  * Verwachtingen
  * Astronomisch getij
 
-De digitale delta api is beschikbaar via:
+De operationele digitale delta api is beschikbaar via:
 
  *  https://ddapi.rws.nl/dd-oper/2.0
 
 De authenticatie op deze url is geregeld via PKI overheidscertificaten.
 Zonder PKI overheidscertificaat is de api niet toegankelijk.
 
-Deze git repository bevat een python module om de dd-oper api te benaderen
+Deze git repository bevat een python module om de dd oper api te benaderen
 inclusief:
 
  * Autenticatie via certificaten.
@@ -24,34 +24,29 @@ inclusief:
 
 ## Installatie
 
-De package is op dit moment nog niet beschikbaar via [PyPI](https://pypi.org/).
-De package is beschikbaar via https://marceln.org/download/python
-
 Het installeren gaat via:
 ```
-pip install -i https://marceln.org/download/python ddOperApi
+pip ddOperApi
 ```
 
 ## De inhoud van de git repository
 
-TODO: Aanpassen aan module
+ * ddOperApi/: De module
+ * ddOperApi/tests: De module test code
+ * githooks: Standaard hooks voor een locale zelf test
+ * demo/DD-API-Oper demo.ipynb: [Jupyter notebook](https://jupyter.org/)
+   demo file
 
- * ddapioper.py: Python module
- * test-ddapi: Een klein voorbeeld en test programma voor de module
- * DD-API-Oper demo.ipynb: Een [Jupyter notebook](met een demo).
-
-### dd-oper-api
+### ddOperApi
 
 Deze module bevat documentatie. Deze is op te vragen door de module te
 laden en vervolgens via de python hulp functie heb je toegang tot de
 documentatie.
 
-### test-ddapi
+### ddOperApi/tests
 
-Een werkend voorbeeld en test programma voor de api.
-
-Wel moeten de paden van het certificaat en de bijbehorende private key
-nog ingevuld worden. Er staat twee keer "<TODO>" in de code.
+Bevat een flask webserver waartegen de module getest kan worden en
+python unittest code. 
 
 ### DD-API-Oper demo.ipynb
 
@@ -85,4 +80,4 @@ stabiele code!
 
 ## Vragen en/of uitbreidingen.
 
-Vragen en of uitbreidingen kunnen gemaild worden naar ddoperapi@pion.xs4all.nl.
+Vragen en of uitbreidingen kunnen gemaild worden naar ddoperapi(a)marceln(.)org.

ddOperApi/tests/__init__.py

@@ -1,23 +0,0 @@
-#!/usr/bin/python
-
-import unittest
-import ddOperApi
-
-class ddOperApiTest(unittest.TestCase):
-
-    def setUp(self):
-        pass
-
-    def test_version(self):
-        self.assertTrue(hasattr(ddOperApi, '__version__'))
-        self.assertEqual(type(ddOperApi.__version__), str)
-
-def setup():
-    pass
-
-def main():
-    setup()
-    unittest.main()
-
-if __name__ == '__main__':
-    main()

githooks/pre-commit

@@ -1,6 +1,10 @@
-#!/bin/sh
-VERSION=$(grep version setup.py | sed -e 's/.*="//' -e 's/",//')
-sed -i "s/^__version__ = .*/__version__ = '${VERSION}'/"  ddOperApi/__init__.py
-git add ddOperApi/__init__.py
+#!/bin/sh -e
 
-python setup.py test
+yamllint .gitea/workflows/build.yml
+
+VERSION=$(python setup.py --version)
+sed -i "s/^__version__ = .*/__version__ = '${VERSION}'/"  src/ddOperApi/__init__.py
+
+git add src/ddOperApi/__init__.py
+
+tox

setup.cfg

@@ -0,0 +1,28 @@
+[metadata]
+name = ddOperApi
+version = 0.9.3
+author = Marcel Nijenhof
+author_email = pypi@marceln.org
+description = Interface for the dd-oper protocol
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://git.marceln.org/marceln/ddOperApi
+classifiers =
+    Programming Language :: Python :: 3
+    License :: OSI Approved :: MIT License
+    Operating System :: OS Independent
+    Development Status :: 4 - Beta
+
+[options]
+package_dir =
+    = src
+install_requires =
+    python-dateutil
+    requests
+    numpy
+    pyopenssl
+packages = find:
+python_requires = >= 3.6
+
+[options.packages.find]
+where = src

setup.py

@@ -1,26 +1,4 @@
-import setuptools
+from setuptools import setup
 
-with open("README.md", "r") as fh:
-    long_description = fh.read()
-
-setuptools.setup(
-    name="ddOperApi", # Replace with your own username
-    version="0.0.3",
-    author="Marcel Nijenhof",
-    author_email="pip@pion.xs4all.nl",
-    description="Interface for dd-oper protocol",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    url="https://marceln.org/git/Werk/lmwsip",
-    packages=setuptools.find_packages(),
-    install_requires=[
-      'python-dateutil',
-    ],
-    classifiers=[
-        "Programming Language :: Python :: 3",
-        "License :: TODO",
-        "Operating System :: OS Independent",
-        "Topic :: dd-oper api" 
-    ],
-    python_requires='>=3.6',
-)
+if __name__ == "__main__":
+    setup()

src/ddOperApi/__init__.py

@@ -17,10 +17,10 @@ import logging
 import requests
 import numpy as np
 from datetime import datetime, timedelta
-from dateutil.parser import isoparse
+from dateutil.parser import parse
 
 """ Version info changed by git hook """
-__version__ = 'XXXX'
+__version__ = '0.9.3'
 
 class ddOperApi:
     """
@@ -62,7 +62,7 @@ Note: Internal use!
 """
         logging.debug("ddOperApi.get(%s)" % url)
         try:
-            req = requests.get(url, cert=(self.certfile, self.certkey))
+            req = requests.get(url, cert=(self.certfile, self.certkey), verify=self.checkssl)
         except Exception as e:
             logging.exception("ddOperApi.get error: %s" % e)
             raise(e)
@@ -272,7 +272,7 @@ Note:
   This may change!
 """
         for e in self.result()["events"]:
-            dt = isoparse(e["timeStamp"])
+            dt = parse(e["timeStamp"])
             if "value" in e:
                 p  = e
             elif "aspects" in e:

tests/main.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python
+
+import unittest
+import ddOperApi
+from time import sleep
+from testServer import forkTestServer, stopTestServer
+
+
+class ddOperApiTest(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        global forkTestServerPid
+        forkTestServerPid = forkTestServer()
+
+    @classmethod
+    def tearDownClass(cls):
+        global forkTestServerPid
+        stopTestServer(forkTestServerPid)
+
+    def setUp(self):
+        self.client = ddOperApi.ddOperApi(url="https://localhost:5000/dd-oper/2.0", checkssl=False)
+
+    def tearDown(self):
+        pass
+
+    def test_version(self):
+        self.assertTrue(hasattr(ddOperApi, '__version__'))
+        self.assertEqual(type(ddOperApi.__version__), str)
+
+    def test_setup(self):
+        self.assertEqual(type(self.client), ddOperApi.ddOperApi)
+
+    def test_locations(self):
+        locations = self.client.locations()
+        self.assertEqual(type(locations), ddOperApi.ddOperLocation)
+        self.assertTrue(len(locations.data) > 0)
+        self.assertEqual(type(locations.data[0]["properties"]["locationName"]), str)
+
+    def test_locationNames(self):
+        names = self.client.locations().locationNames()
+        self.assertEqual(type(names), type({}.keys()))
+        self.assertTrue(len(names) > 0)
+        for n in names:
+            self.assertEqual(type(n), str)
+
+    def test_locationDetail(self):
+        locations = self.client.locations()
+        name = locations.data[0]["properties"]["locationName"]
+        detail = locations.locationDetail(name)
+        self.assertEqual(type(detail), dict)
+        self.assertEqual(detail["properties"]["locationName"], name)
+
+    def test_quantities(self):
+        quantities = self.client.quantities()
+        self.assertEqual(type(quantities), ddOperApi.ddOperQuantitie)
+        self.assertTrue(len(quantities.data) > 0)
+        self.assertEqual(type(quantities.data[0]), str)
+        for q in quantities.quantities():
+            self.assertEqual(type(q), str)
+
+    def test_values(self):
+        values = self.client.values("test1", "null")
+        self.assertEqual(type(values), ddOperApi.ddOperValues)
+
+def main():
+    unittest.main()
+
+if __name__ == '__main__':
+    main()

tests/testServer.py

@@ -0,0 +1,249 @@
+#!/usr/bin/python
+
+import json
+
+from os import fork, kill
+from time import sleep
+from random import getrandbits
+from tempfile import NamedTemporaryFile
+from datetime import datetime, timedelta
+from dateutil.parser import parse
+from dateutil.tz import gettz
+from OpenSSL import crypto, SSL
+from flask import Flask, request
+
+#
+# TODO: createCert verplaatsen zodat die ook beschikbaar is in de module.
+#       Dan kan deze als client certificaat gebruikt worden en kan er
+#       getest worden met client certificaat.
+#       De extra subclass kan dan ook weg!
+#
+
+class testServer(Flask):
+    def run(self):
+        self.createCert()
+        self.writeCert()
+        super().run(ssl_context=(self.certFile.name, self.keyFile.name))
+
+    def createCert(self):
+        CN="*"
+
+        k = crypto.PKey()
+        k.generate_key(crypto.TYPE_RSA, 2048)
+        serialnumber=getrandbits(64)
+
+        cert = crypto.X509()
+        cert.get_subject().C = "TS"
+        cert.get_subject().L = "Test"
+        cert.get_subject().ST = "Test"
+        cert.get_subject().O = "Test"
+        cert.get_subject().OU = "Test"
+        cert.get_subject().CN = CN
+        cert.set_serial_number(serialnumber)
+        cert.gmtime_adj_notBefore(0)
+        cert.gmtime_adj_notAfter(3600)
+        cert.set_issuer(cert.get_subject())
+        cert.add_extensions([crypto.X509Extension(b'subjectAltName', False, b'DNS:*')])
+        cert.set_pubkey(k)
+        cert.sign(k, 'sha512')
+        self.cert=crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+        self.key=crypto.dump_privatekey(crypto.FILETYPE_PEM, k)
+
+    def writeCert(self):
+        self.certFile = NamedTemporaryFile()
+        self.certFile.file.write(self.cert)
+        self.certFile.file.flush()
+
+        self.keyFile  = NamedTemporaryFile()
+        self.keyFile.file.write(self.key)
+        self.keyFile.flush()
+
+app = testServer(__name__)
+
+def timeStr(dt):
+    return("%sz" % dt.isoformat())
+
+def provider(responseType):
+    return({
+        "name": "testServerDDOper",
+        "supportUrl": "https://git.marceln.org/marceln/ddOperApi",
+        "responseType": responseType,
+        "responseTimestamp": timeStr(datetime.now())
+    })
+
+def geometryPoint(x, y):
+    return({
+        "type": "Point",
+        "coordinates": [ x, y ]
+    })
+
+def roundTime(startTime, intervalSec):
+    t = startTime.timestamp()
+    m = intervalSec
+    if (t%m > 0):
+        s = int(t/m+1)*m
+        return(datetime.fromtimestamp(s, tz=gettz("UTC")))
+    else:
+        return(startTime)
+
+def singleEvent(eventTime, intervalSec, location, quantity, aspectSet):
+    return({
+        "timeStamp": timeStr(eventTime),
+        "startTime": timeStr(eventTime-timedelta(seconds=intervalSec/2)),
+        "endTime": timeStr(eventTime+timedelta(seconds=intervalSec/2)),
+        "quality": 10,
+        "value": 0,
+        "additionalInfo": 0
+    })
+            
+def events(location, quantity, aspectSet, intervalLength, startTime, endTime):
+    ret = []
+    if intervalLength == "10min":
+        intervalSec=600
+    elif intervalLength == "1min":
+        intervalSec=60
+    eventTime = roundTime(startTime, intervalSec)
+    while (eventTime < endTime):
+        ret.append(singleEvent(eventTime, intervalSec, location, quantity, aspectSet))
+        eventTime = eventTime + timedelta(seconds=intervalSec)
+    return(ret)
+
+def jsonReturn(data):
+    return(json.dumps(data))
+
+@app.route("/")
+def root():
+    return("Hello world!\n")
+
+@app.route("/dd-oper/2.0/locations")
+def locations():
+    ret = { "provider": provider("LocationListResponse"),
+            "results": [
+                {
+                    "type": "Feature",
+                    "geometry": geometryPoint(0, 0),
+                    "properties": {
+                        "locationName": "test1",
+                        "locationNameSpace": "TS.DDOPER",
+                        "displayNameGlobal": "Test location 1"
+                    }
+                },
+                {
+                    "type": "Feature",
+                    "geometry": geometryPoint(1, 0),
+                    "properties": {
+                        "locationName": "test2",
+                        "locationNameSpace": "TS.DDOPER",
+                        "displayNameGlobal": "Test location 2"
+                    }
+                },
+                {
+                    "type": "Feature",
+                    "geometry": geometryPoint(0, 1),
+                    "properties": {
+                        "locationName": "test3",
+                        "locationNameSpace": "TS.DDOPER",
+                        "displayNameGlobal": "Test location 3"
+                    }
+                }
+            ],
+            "_comment": "MKK-DL: Location catalogue"
+          }
+    return(jsonReturn(ret))
+
+@app.route("/dd-oper/2.0/quantities")
+def quantities():
+    ret = { "provider": provider("QuantityListResponse"),
+            "results": [
+                "time",
+                "null"
+            ],
+            "_comment": "MKK-DL: Quantities catalogue"
+    }
+    return(jsonReturn(ret))
+
+@app.route("/dd-oper/2.0/locations/<location>/quantities")
+def locationsQuantities(location):
+    return("TODO: locations: %s\n" % location, 404)
+
+@app.route("/dd-oper/2.0/quantities/<quantity>")
+def quantitiesLocations(quantity):
+    return("TODO: quantities: %s\n" % quantity, 404)
+
+@app.route("/dd-oper/2.0/locations/<location>/quantities/<quantity>/timeseries")
+def values(location, quantity):
+    startTime       = request.args.get('startTime', type = str)
+    endTime         = request.args.get('endTime', type = str)
+    process         = request.args.get('process', default='measurement', type = str)
+    intervalLength  = request.args.get('intervalLength', default='10min', type = str)
+    aspectSet       = request.args.get('aspectSet', default="minimum", type = str)
+    try:
+        startTime = parse(startTime)
+        endTime   = parse(endTime)
+    except Exception as e:
+        return("parseerror time", 404)
+    ret = { "provider": provider("Timeseries"),
+            "results": [{
+                "source": {
+                    "process": process,
+                    "institution": {
+                        "name": "testServerDDOper"
+                    },
+                    "location": {
+                        "geometry": {
+                            "type": "Point",
+                            "coordinates": [ 0, 0]
+                        }
+                    },
+                    "type": "Feature",
+                    "properties": {
+                        "locationName": location,
+                        "locationNameSpace": "TS.DDOPER",
+                        "crsName": "WGS84",
+                        "displayNameGlobal": "test 1"
+                    }
+                },
+                "startTime": timeStr(startTime),
+                "endTime":   timeStr(endTime),
+                "events":    events(location, quantity, aspectSet, intervalLength, startTime, endTime),
+                "observationType": {
+                    "quantityName": quantity,
+                    "aspectSet": {
+                        "name": "minimum",
+                        "aspects": [
+                            {
+                                "name": "Average",
+                                "unit": ""
+                            }
+                        ]
+                    }
+                }
+            }]
+          }
+    return(jsonReturn(ret))
+
+def forkTestServer():
+    pid = None
+    try:
+        pid = fork()
+        if (pid == 0):
+            app.run()
+        else:
+            sleep(1)
+    except Exception as e:
+        raise(e)
+    return(pid)
+
+def stopTestServer(pid):
+    try:
+        kill(pid, 15)
+        sleep(0.01)
+    except Exception as e:
+        print("ERROR kill: %s" % e)
+    return()
+
+def main():
+    app.run()
+
+if __name__ == '__main__':
+    main()

tox.ini

@@ -0,0 +1,6 @@
+[tox]
+envlist = py38,py39,py310,py311,py312
+
+[testenv]
+deps = flask
+commands = python tests/main.py