Versie als module & tests

.drone.yml

@@ -1,27 +1,170 @@
 ---
 kind: pipeline
 type: exec
-name: default
+name: CentOS7_test
+
 platform:
   os: linux
   arch: amd64
+  variant: CentOS7
+
 steps:
-- name: Run unit test
-  environment:
-   PYTHONPATH: .
-  commands:
-  - python setup.py test
-- name: Build package files
-  commands:
-  - python setup.py sdist
-- name: Versie toevoegen aan download
-  environment:
-    DOWNLOADDIR: /usr/share/nginx/html/download/python/ddoperapi
-  commands:
-  - mkdir -p "$${DOWNLOADDIR}"
-  - cd dist
-  - for f in *;do if [ -f "$${DOWNLOADDIR}/$${f}" ] ; then echo version error;exit 1;else cp "$${f}" "$${DOWNLOADDIR}";fi;done
-  - ls -l "$${DOWNLOADDIR}"
-  when:
-    branch:
-    - master
+  - name: Run unit test
+    environment:
+      PYTHONPATH: .
+    commands:
+      - python3 setup.py test
+
+---
+kind: pipeline
+type: exec
+name: CentOS8_test
+
+platform:
+  os: linux
+  arch: amd64
+  variant: CentOS8
+
+steps:
+  - name: Run unit test
+    environment:
+      PYTHONPATH: .
+    commands:
+      - python3 setup.py test
+
+---
+kind: pipeline
+type: exec
+name: Ubuntu1804_test
+
+platform:
+  os: linux
+  arch: amd64
+  variant: Ubuntu1804
+
+steps:
+  - name: Run unit test
+    environment:
+      PYTHONPATH: .
+    commands:
+      - python3 setup.py test
+
+---
+kind: pipeline
+type: exec
+name: Fedora_test
+
+platform:
+  os: linux
+  arch: amd64
+  variant: Fedora
+
+steps:
+  - name: Run unit test
+    environment:
+      PYTHONPATH: .
+    commands:
+      - python3 setup.py test
+
+---
+kind: pipeline
+type: docker
+name: docker_python:3.6
+
+steps:
+  - name: Run unit test
+    image: python:3.6
+    environment:
+      PYTHONPATH: .
+    commands:
+      - pip install python-dateutil flask
+      - python setup.py test
+
+---
+kind: pipeline
+type: docker
+name: docker_python:latest
+
+steps:
+  - name: Run unit test
+    image: python:latest
+    environment:
+      PYTHONPATH: .
+    commands:
+      - pip install python-dateutil flask
+      - python setup.py test
+
+---
+kind: pipeline
+type: exec
+name: Build
+
+platform:
+  os: linux
+  arch: amd64
+  vatiant: Fedora
+
+steps:
+  - name: Build package files
+    commands:
+      - python setup.py sdist
+  - name: Versie toevoegen aan download
+    environment:
+      DOWNLOADDIR: /usr/share/nginx/html/download/python/ddoperapi
+    commands:
+      - mkdir -p "$${DOWNLOADDIR}"
+      - cd dist
+      - |
+        for f in *
+        do
+          if [ -f "$${DOWNLOADDIR}/$${f}" ]
+          then
+            echo version error
+            exit 1
+          else
+            cp "$${f}" "$${DOWNLOADDIR}"
+          fi
+        done
+      - ls -l "$${DOWNLOADDIR}"
+    when:
+      branch:
+        - master
+
+depends_on:
+  - CentOS7_test
+  - CentOS8_test
+  - Ubuntu1804_test
+  - Fedora_test
+  - docker_python:latest
+  - docker_python:3.6
+
+---
+kind: pipeline
+type: docker
+name: installCheck
+
+steps:
+  - name: install Check
+    image: python:3.6
+    # Make sure we run the pip installed version
+    commands:
+      - rm -rf ddOperApi
+    when:
+      branch:
+        - master
+
+depends_on:
+  - Build
+
+#
+# todo: Uitzoeken
+#
+#      - >
+#        pip install --extra-index-url
+#        https://marceln.org/download/python ddOperApi
+#      - python -c "import lmwsip"
+#      - python -c "import lmwsip; print(lmwsip.__version__)"
+#      - >
+#       [ $(python -c "import lmwsip; print(lmwsip.__version__)") =
+#       $(python setup.py --version) ]
+#      - python -m unittest -v lmwsip.tests

.gitignore

@@ -2,3 +2,8 @@
 __pycache__
 dd_oper_api.egg-info
 dist
+dependency_links.txt
+PKG-INFO
+requires.txt
+SOURCES.txt
+top_level.txt

ddOperApi/__init__.py

@@ -17,10 +17,10 @@ import logging
 import requests
 import numpy as np
 from datetime import datetime, timedelta
-from dateutil.parser import isoparse
+from dateutil.parser import parse
 
 """ Version info changed by git hook """
-__version__ = 'XXXX'
+__version__ = '0.1.0'
 
 class ddOperApi:
     """
@@ -62,7 +62,7 @@ Note: Internal use!
 """
         logging.debug("ddOperApi.get(%s)" % url)
         try:
-            req = requests.get(url, cert=(self.certfile, self.certkey))
+            req = requests.get(url, cert=(self.certfile, self.certkey), verify=self.checkssl)
         except Exception as e:
             logging.exception("ddOperApi.get error: %s" % e)
             raise(e)
@@ -272,7 +272,7 @@ Note:
   This may change!
 """
         for e in self.result()["events"]:
-            dt = isoparse(e["timeStamp"])
+            dt = parse(e["timeStamp"])
             if "value" in e:
                 p  = e
             elif "aspects" in e:

ddOperApi/tests/__init__.py

@@ -2,21 +2,68 @@
 
 import unittest
 import ddOperApi
+from time import sleep
+from ddOperApi.tests.testServer import forkTestServer, stopTestServer
+
 
 class ddOperApiTest(unittest.TestCase):
 
+    @classmethod
+    def setUpClass(cls):
+        global forkTestServerPid
+        forkTestServerPid = forkTestServer()
+
+    @classmethod
+    def tearDownClass(cls):
+        global forkTestServerPid
+        stopTestServer(forkTestServerPid)
+
     def setUp(self):
+        self.client = ddOperApi.ddOperApi(url="https://localhost:5000/dd-oper/2.0", checkssl=False)
+
+    def tearDown(self):
         pass
 
     def test_version(self):
         self.assertTrue(hasattr(ddOperApi, '__version__'))
         self.assertEqual(type(ddOperApi.__version__), str)
 
-def setup():
-    pass
+    def test_setup(self):
+        self.assertEqual(type(self.client), ddOperApi.ddOperApi)
+
+    def test_locations(self):
+        locations = self.client.locations()
+        self.assertEqual(type(locations), ddOperApi.ddOperLocation)
+        self.assertTrue(len(locations.data) > 0)
+        self.assertEqual(type(locations.data[0]["properties"]["locationName"]), str)
+
+    def test_locationNames(self):
+        names = self.client.locations().locationNames()
+        self.assertEqual(type(names), type({}.keys()))
+        self.assertTrue(len(names) > 0)
+        for n in names:
+            self.assertEqual(type(n), str)
+
+    def test_locationDetail(self):
+        locations = self.client.locations()
+        name = locations.data[0]["properties"]["locationName"]
+        detail = locations.locationDetail(name)
+        self.assertEqual(type(detail), dict)
+        self.assertEqual(detail["properties"]["locationName"], name)
+
+    def test_quantities(self):
+        quantities = self.client.quantities()
+        self.assertEqual(type(quantities), ddOperApi.ddOperQuantitie)
+        self.assertTrue(len(quantities.data) > 0)
+        self.assertEqual(type(quantities.data[0]), str)
+        for q in quantities.quantities():
+            self.assertEqual(type(q), str)
+
+    def test_values(self):
+        values = self.client.values("test1", "null")
+        self.assertEqual(type(values), ddOperApi.ddOperValues)
 
 def main():
-    setup()
     unittest.main()
 
 if __name__ == '__main__':

ddOperApi/tests/testServer.py

@@ -0,0 +1,249 @@
+#!/usr/bin/python
+
+import json
+
+from os import fork, kill
+from time import sleep
+from random import getrandbits
+from tempfile import NamedTemporaryFile
+from datetime import datetime, timedelta
+from dateutil.parser import parse
+from dateutil.tz import gettz
+from OpenSSL import crypto, SSL
+from flask import Flask, request
+
+#
+# TODO: createCert verplaatsen zodat die ook beschikbaar is in de module.
+#       Dan kan deze als client certificaat gebruikt worden en kan er
+#       getest worden met client certificaat.
+#       De extra subclass kan dan ook weg!
+#
+
+class testServer(Flask):
+    def run(self):
+        self.createCert()
+        self.writeCert()
+        super().run(ssl_context=(self.certFile.name, self.keyFile.name))
+
+    def createCert(self):
+        CN="*"
+
+        k = crypto.PKey()
+        k.generate_key(crypto.TYPE_RSA, 2048)
+        serialnumber=getrandbits(64)
+
+        cert = crypto.X509()
+        cert.get_subject().C = "TS"
+        cert.get_subject().L = "Test"
+        cert.get_subject().ST = "Test"
+        cert.get_subject().O = "Test"
+        cert.get_subject().OU = "Test"
+        cert.get_subject().CN = CN
+        cert.set_serial_number(serialnumber)
+        cert.gmtime_adj_notBefore(0)
+        cert.gmtime_adj_notAfter(3600)
+        cert.set_issuer(cert.get_subject())
+        cert.add_extensions([crypto.X509Extension(b'subjectAltName', False, b'DNS:*')])
+        cert.set_pubkey(k)
+        cert.sign(k, 'sha512')
+        self.cert=crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+        self.key=crypto.dump_privatekey(crypto.FILETYPE_PEM, k)
+
+    def writeCert(self):
+        self.certFile = NamedTemporaryFile()
+        self.certFile.file.write(self.cert)
+        self.certFile.file.flush()
+
+        self.keyFile  = NamedTemporaryFile()
+        self.keyFile.file.write(self.key)
+        self.keyFile.flush()
+
+app = testServer(__name__)
+
+def timeStr(dt):
+    return("%sz" % dt.isoformat())
+
+def provider(responseType):
+    return({
+        "name": "testServerDDOper",
+        "supportUrl": "https://git.marceln.org/marceln/ddOperApi",
+        "responseType": responseType,
+        "responseTimestamp": timeStr(datetime.now())
+    })
+
+def geometryPoint(x, y):
+    return({
+        "type": "Point",
+        "coordinates": [ x, y ]
+    })
+
+def roundTime(startTime, intervalSec):
+    t = startTime.timestamp()
+    m = intervalSec
+    if (t%m > 0):
+        s = int(t/m+1)*m
+        return(datetime.fromtimestamp(s, tz=gettz("UTC")))
+    else:
+        return(startTime)
+
+def singleEvent(eventTime, intervalSec, location, quantity, aspectSet):
+    return({
+        "timeStamp": timeStr(eventTime),
+        "startTime": timeStr(eventTime-timedelta(seconds=intervalSec/2)),
+        "endTime": timeStr(eventTime+timedelta(seconds=intervalSec/2)),
+        "quality": 10,
+        "value": 0,
+        "additionalInfo": 0
+    })
+            
+def events(location, quantity, aspectSet, intervalLength, startTime, endTime):
+    ret = []
+    if intervalLength == "10min":
+        intervalSec=600
+    elif intervalLength == "1min":
+        intervalSec=60
+    eventTime = roundTime(startTime, intervalSec)
+    while (eventTime < endTime):
+        ret.append(singleEvent(eventTime, intervalSec, location, quantity, aspectSet))
+        eventTime = eventTime + timedelta(seconds=intervalSec)
+    return(ret)
+
+def jsonReturn(data):
+    return(json.dumps(data))
+
+@app.route("/")
+def root():
+    return("Hello world!\n")
+
+@app.route("/dd-oper/2.0/locations")
+def locations():
+    ret = { "provider": provider("LocationListResponse"),
+            "results": [
+                {
+                    "type": "Feature",
+                    "geometry": geometryPoint(0, 0),
+                    "properties": {
+                        "locationName": "test1",
+                        "locationNameSpace": "TS.DDOPER",
+                        "displayNameGlobal": "Test location 1"
+                    }
+                },
+                {
+                    "type": "Feature",
+                    "geometry": geometryPoint(1, 0),
+                    "properties": {
+                        "locationName": "test2",
+                        "locationNameSpace": "TS.DDOPER",
+                        "displayNameGlobal": "Test location 2"
+                    }
+                },
+                {
+                    "type": "Feature",
+                    "geometry": geometryPoint(0, 1),
+                    "properties": {
+                        "locationName": "test3",
+                        "locationNameSpace": "TS.DDOPER",
+                        "displayNameGlobal": "Test location 3"
+                    }
+                }
+            ],
+            "_comment": "MKK-DL: Location catalogue"
+          }
+    return(jsonReturn(ret))
+
+@app.route("/dd-oper/2.0/quantities")
+def quantities():
+    ret = { "provider": provider("QuantityListResponse"),
+            "results": [
+                "time",
+                "null"
+            ],
+            "_comment": "MKK-DL: Quantities catalogue"
+    }
+    return(jsonReturn(ret))
+
+@app.route("/dd-oper/2.0/locations/<location>/quantities")
+def locationsQuantities(location):
+    return("TODO: locations: %s\n" % location, 404)
+
+@app.route("/dd-oper/2.0/quantities/<quantity>")
+def quantitiesLocations(quantity):
+    return("TODO: quantities: %s\n" % quantity, 404)
+
+@app.route("/dd-oper/2.0/locations/<location>/quantities/<quantity>/timeseries")
+def values(location, quantity):
+    startTime       = request.args.get('startTime', type = str)
+    endTime         = request.args.get('endTime', type = str)
+    process         = request.args.get('process', default='measurement', type = str)
+    intervalLength  = request.args.get('intervalLength', default='10min', type = str)
+    aspectSet       = request.args.get('aspectSet', default="minimum", type = str)
+    try:
+        startTime = parse(startTime)
+        endTime   = parse(endTime)
+    except Exception as e:
+        return("parseerror time", 404)
+    ret = { "provider": provider("Timeseries"),
+            "results": [{
+                "source": {
+                    "process": process,
+                    "institution": {
+                        "name": "testServerDDOper"
+                    },
+                    "location": {
+                        "geometry": {
+                            "type": "Point",
+                            "coordinates": [ 0, 0]
+                        }
+                    },
+                    "type": "Feature",
+                    "properties": {
+                        "locationName": location,
+                        "locationNameSpace": "TS.DDOPER",
+                        "crsName": "WGS84",
+                        "displayNameGlobal": "test 1"
+                    }
+                },
+                "startTime": timeStr(startTime),
+                "endTime":   timeStr(endTime),
+                "events":    events(location, quantity, aspectSet, intervalLength, startTime, endTime),
+                "observationType": {
+                    "quantityName": quantity,
+                    "aspectSet": {
+                        "name": "minimum",
+                        "aspects": [
+                            {
+                                "name": "Average",
+                                "unit": ""
+                            }
+                        ]
+                    }
+                }
+            }]
+          }
+    return(jsonReturn(ret))
+
+def forkTestServer():
+    pid = None
+    try:
+        pid = fork()
+        if (pid == 0):
+            app.run()
+        else:
+            sleep(1)
+    except Exception as e:
+        raise(e)
+    return(pid)
+
+def stopTestServer(pid):
+    try:
+        kill(pid, 15)
+        sleep(0.01)
+    except Exception as e:
+        print("ERROR kill: %s" % e)
+    return()
+
+def main():
+    app.run()
+
+if __name__ == '__main__':
+    main()

githooks/pre-commit

@@ -1,6 +1,7 @@
-#!/bin/sh
+#!/bin/sh -e
 VERSION=$(grep version setup.py | sed -e 's/.*="//' -e 's/",//')
 sed -i "s/^__version__ = .*/__version__ = '${VERSION}'/"  ddOperApi/__init__.py
 git add ddOperApi/__init__.py
 
 python setup.py test
+yamllint .drone.yml

setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="ddOperApi", # Replace with your own username
-    version="0.0.3",
+    version="0.1.0",
     author="Marcel Nijenhof",
     author_email="pip@pion.xs4all.nl",
     description="Interface for dd-oper protocol",
@@ -15,6 +15,9 @@ setuptools.setup(
     packages=setuptools.find_packages(),
     install_requires=[
       'python-dateutil',
+      'requests',
+      'numpy',
+      'pyopenssl'
     ],
     classifiers=[
         "Programming Language :: Python :: 3",