From 889154ac2d49cb54be9250413214f221e28386b2 Mon Sep 17 00:00:00 2001 From: rowan04 Date: Thu, 12 Feb 2026 13:15:01 +0000 Subject: [PATCH 1/5] Update HTML and request/context variables for apel loader check - has radio buttons to specifiy whether record should be validated or loaded --- .../templates/validator/validator_index.html | 16 +++++++++++++--- monitoring/validator/views.py | 9 ++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/monitoring/validator/templates/validator/validator_index.html b/monitoring/validator/templates/validator/validator_index.html index 748741a..4d04b18 100644 --- a/monitoring/validator/templates/validator/validator_index.html +++ b/monitoring/validator/templates/validator/validator_index.html @@ -41,18 +41,28 @@

Record Validator

wrap="wrap" >{{ input_record|default:''|escape }}

- +

Select 'Validate' if you want to check only the formatting of the record(s).

+

Select 'Load' if you want to also check whether the record(s) will successfully load into the database.

+ Validate +
+ Load +


+

{% if output %} -

Validation Output

+

Submission Output

{{ output }} {% endif %}
diff --git a/monitoring/validator/views.py b/monitoring/validator/views.py index 1490d8a..bc541e7 100644 --- a/monitoring/validator/views.py +++ b/monitoring/validator/views.py @@ -26,17 +26,24 @@ def index(request): template_name = "validator/validator_index.html" input_record = "" record_type = "All" + submission_type = "" output = "" # On form submission, trigger record validation if request.method == "POST": input_record = request.POST.get("input_record", "") record_type = request.POST.get("record_type", "") - output = validate(input_record, record_type) + submission_type = request.POST.get("submission_type", "") + if submission_type == "load": + #output = load(input_record, submission_type) + output = "loaded" + else: + output = validate(input_record, record_type) context = { "input_record": input_record, "record_type": record_type, + "submission_type": submission_type, "output": output, } From 750ec66146e23e327396b504c8f95a8a6e068521 Mon Sep 17 00:00:00 2001 From: rowan04 Date: Thu, 12 Feb 2026 18:14:03 +0000 Subject: [PATCH 2/5] Add database loading check to monitoring validator - adds new load method - creates instance of apel loader - makes use of a blackhole mariadb database, which doesn't store data --- monitoring/validator/views.py | 90 ++++++++++++++++++++++++++------- monitoring/validatorSettings.py | 33 ++++++++++++ 2 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 monitoring/validatorSettings.py diff --git a/monitoring/validator/views.py b/monitoring/validator/views.py index bc541e7..2e643b7 100644 --- a/monitoring/validator/views.py +++ b/monitoring/validator/views.py @@ -1,7 +1,8 @@ from django.shortcuts import render from django.views.decorators.http import require_http_methods -# Apel record-checking class imports +# Apel loader and record-checking class imports +from apel.db.loader.loader import Loader from apel.db.loader.record_factory import RecordFactory, RecordFactoryException from apel.db.records.record import InvalidRecordException @@ -13,32 +14,51 @@ from apel.db.records.cloud import CloudRecord from apel.db.records.cloud_summary import CloudSummaryRecord +# Validator db configuration +from monitoring import validatorSettings + +# Python tempfile for temporary apel queues +import tempfile + @require_http_methods(["GET", "POST"]) def index(request): """ - Validates inputted records using the Apel record validation methods. - It either validates a record against a specific type or against all types, depending on what record_type + Validates inputted records using the Apel record validation methods, or tests they load correctly using an + instance of the Apel loader. + For validation: + It either validates a record against a specific type or against all types, depending on what record_type option was chosen on the html template. The default is `All`. - The input record, record type and validation output are then returned to the html template as context on get - request, so that the html page retains its information/context when refreshing the page or submitting the form. + For loading: + It both validates the record's syntax against all types, and checks it can load into a database correctly + using the Apel loader class. The database uses a blackhole engine, so no data is stored. + The input record, record type, submission type and validation output are then returned to the html template + as context on get request, so that the html page retains its information/context when refreshing the page + or submitting the form. """ + template_name = "validator/validator_index.html" input_record = "" record_type = "All" submission_type = "" output = "" - # On form submission, trigger record validation + # On form submission, check record isnt empty. + # Then trigger record validation or record loading, based on submission type if request.method == "POST": input_record = request.POST.get("input_record", "") record_type = request.POST.get("record_type", "") submission_type = request.POST.get("submission_type", "") - if submission_type == "load": - #output = load(input_record, submission_type) - output = "loaded" + + if input_record: + input_record = input_record.strip() + + if submission_type == "load": + output = load(input_record) + else: + output = validate(input_record, record_type) else: - output = validate(input_record, record_type) + output = "Please enter a record to be validated." context = { "input_record": input_record, @@ -52,17 +72,13 @@ def index(request): def validate(record: str, record_type: str) -> str: """ - Validated record(s) and record_type passed in from the html page template. - If record type is all, make use of the create_records apel method (expects a record header). - Else, make use of the _create_record_objects apel method (expects there to be no record header). + Record(s) and record_type passed in from the html page template. + If record type is all, make use of the `create_records` apel method (expects a record header). + Else, make use of the `_create_record_objects` apel method (expects there to be no record header). If the record is valid, return a "valid record" string. If the record is invalid, an InvalidRecordException or RecordFactoryException is raised by the Apel methods. Catch these exceptions and return the exception information. """ - if not record: - return "Please enter a record to be validated." - - record = record.strip() # Map record_type string to record_type class # String is always exact as determined through html form selection option @@ -94,3 +110,43 @@ def validate(record: str, record_type: str) -> str: except (InvalidRecordException, RecordFactoryException) as e: return str(e) + +def load(record: str) -> str: + """ + Record passed in from the html page template. + Create a loader instance, making use of a tempfile directory for the queues and pidfile creation + Startup loader and then pass record(s) and blank signer into the `load_msg` method + Make use of a blackhole database, which loads but doesn't store any data. + If the record loads successfully, return a "valid record" string. + If the record fails to load, an exception is raised by the Apel methods. + Catch these exceptions and return the exception information. + """ + + try: + validatorDB = validatorSettings.VALIDATOR_DB + + # Set the tempfile temporary directory within /tmp + tempfile.tempdir = "/tmp" + + qpath = tempfile.gettempdir() + db_backend = validatorDB.get("ENGINE") + db_host = validatorDB.get("HOST") + db_port = int(validatorDB.get("PORT")) + db_name = validatorDB.get("NAME") + db_username = validatorDB.get("USER") + db_password = validatorDB.get("PASSWORD") + pidfile = "" + signer = "" + + loader = Loader(qpath, record, db_backend, db_host, db_port, db_name, db_username, db_password, pidfile) + + loader.startup() + + loader.load_msg(record, signer) + + loader.shutdown() + + return("Record(s) will load successfully!") + + except Exception as e: + return str(e) diff --git a/monitoring/validatorSettings.py b/monitoring/validatorSettings.py new file mode 100644 index 0000000..7194ba8 --- /dev/null +++ b/monitoring/validatorSettings.py @@ -0,0 +1,33 @@ +""" +Settings for the validator app, part of the monitoring project. + +These settings are for the apel modules, so they need to be kept separate +from Django's control. Because Django was having issues with some of +the defined configuration, such as the database engine. +""" + +import configparser +import os +import sys + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +try: + # Read configuration from the file + cp = configparser.ConfigParser(interpolation=None) + file_path = os.path.join(BASE_DIR, 'monitoring', 'settings.ini') + cp.read(file_path) + + VALIDATOR_DB = { + 'ENGINE': cp.get('db_validator', 'backend'), + 'HOST': cp.get('db_validator', 'hostname'), + 'PORT': cp.get('db_validator', 'port'), + 'NAME': cp.get('db_validator', 'name'), + 'USER': cp.get('db_validator', 'username'), + 'PASSWORD': cp.get('db_validator', 'password'), + } + +except (configparser.NoSectionError) as err: + print("Error in configuration file. Check that file exists first: %s" % err) + sys.exit(1) From bc8342eff93b2f8c3d927ec4840a67f06c3689a9 Mon Sep 17 00:00:00 2001 From: rowan04 Date: Thu, 12 Feb 2026 18:29:40 +0000 Subject: [PATCH 3/5] Improve loader check exception handling, using specific exceptions - also fixes validator bug which would prevent v0.4 records from getting a formatted output --- monitoring/validator/views.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/monitoring/validator/views.py b/monitoring/validator/views.py index 2e643b7..5b0fb64 100644 --- a/monitoring/validator/views.py +++ b/monitoring/validator/views.py @@ -2,7 +2,8 @@ from django.views.decorators.http import require_http_methods # Apel loader and record-checking class imports -from apel.db.loader.loader import Loader +from apel.db import ApelDbException +from apel.db.loader.loader import Loader, LoaderException from apel.db.loader.record_factory import RecordFactory, RecordFactoryException from apel.db.records.record import InvalidRecordException @@ -103,7 +104,7 @@ def validate(record: str, record_type: str) -> str: record_class = record_map[record_type] result = recordFactory._create_record_objects(record, record_class) - if "Record object at" in str(result): + if "object at" in str(result): return "Record(s) valid!" return str(result) @@ -148,5 +149,5 @@ def load(record: str) -> str: return("Record(s) will load successfully!") - except Exception as e: + except (ApelDbException, InvalidRecordException, LoaderException, RecordFactoryException) as e: return str(e) From b68391e65b98174caccd950e2392dbb532ebacdc Mon Sep 17 00:00:00 2001 From: rowan04 Date: Tue, 24 Feb 2026 17:39:33 +0000 Subject: [PATCH 4/5] Fix changes identified in code review - removes unnecessary and duplicated html id attributes - removes apel from requirements.txt --- monitoring/validator/templates/validator/validator_index.html | 4 ++-- requirements.txt | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/monitoring/validator/templates/validator/validator_index.html b/monitoring/validator/templates/validator/validator_index.html index 4d04b18..031266a 100644 --- a/monitoring/validator/templates/validator/validator_index.html +++ b/monitoring/validator/templates/validator/validator_index.html @@ -43,9 +43,9 @@

Record Validator



Select 'Validate' if you want to check only the formatting of the record(s).

Select 'Load' if you want to also check whether the record(s) will successfully load into the database.

- Validate + Validate
- Load + Load




diff --git a/requirements.txt b/requirements.txt index b8bae58..69f5922 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ # Pin packages to support and work with py3.6. -apel Django==3.2.25 djangorestframework==3.15.1 pytz==2025.2 From d6c26681f61de3179e604b1ab44ec478da0bf216 Mon Sep 17 00:00:00 2001 From: rowan04 Date: Wed, 25 Feb 2026 10:01:56 +0000 Subject: [PATCH 5/5] Add documentation for setting up a blackhole engined database --- docs/blackhole_db_setup.md | 84 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 docs/blackhole_db_setup.md diff --git a/docs/blackhole_db_setup.md b/docs/blackhole_db_setup.md new file mode 100644 index 0000000..c0c1bf3 --- /dev/null +++ b/docs/blackhole_db_setup.md @@ -0,0 +1,84 @@ +## Steps to set up a blackhole database for local testing of the loading validator + +As part of the validator app, records are sent to a local instance of the apel +loader class, to test if they load into an apel server database correctly. +This is important as some errors in a record won't be detected by the syntax +validator, but will still cause a record to fail to load. + +These records are loaded into a database with blackhole engined tables - these +tables allow insert commands, but don't store any rows or data, as data is +discarded on write. This allows complete checking that a record can be +successfully loaded to a database, without having to deal with data being stored. + +The local instance of the apel loader class is within `monitoring/views.py`, and +uses `monitoring/validatorSettings.py` to pull configuration settings from +`monitoring/settings.ini` about the blackhole validator database. + +Steps to set up a blackhole-engined version of the apel server database: + +1. Ensure maraidb is started and enabled: + - `sudo su` + - `sudo systemctl start mariadb` + - `sudo systemctl enable mariadb` + +2. Login to mariadb with root: + - `mysql -u root -p` + +3. Install the blackhole plugin, and then verify it is installed: + - `INSTALL SONAME 'ha_blackhole';` + - `SHOW ENGINES;` (should be a row with BLACKHOLE and support as YES). + +4. Exit mariadb: + - `exit;` + +5. Set the global default storage engine to blackhole, so that when the database + schema gets applied, tables are created with the blackhole engine: + - find where your mariadb settings are stored (for me it was `/etc/my.cnf.d/`). + - either edit `server.cnf` or create a new `blackhole.cnf` file (what I did). + - in that file: + ``` + [mysqld] + default-storage-engine=BLACKHOLE + ``` + - This config setting means that any create table statements without an engine + defined will be set to a blackhole engine by default. + +6. Restart mariadb: + - `sudo systemctl restart mariadb` + - Running `SHOW ENGINES;` within mariadb at this point should show the Blackhole + row with support as DEFAULT; + +7. Create the database: + - `mysql -u root -p` + - `CREATE DATABASE validator_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;` + - `CREATE USER 'your_name'@'localhost' IDENTIFIED BY 'your_password';` + - `GRANT ALL PRIVILEGES ON validator_db.* TO 'your_name'@'localhost';` + - `FLUSH PRIVILEGES;` + - `exit;` + +8. Apply the apel server schema to the database: + - the schema is at https://github.com/apel/apel/blob/dev/schemas/server.sql. + - to do this step, I used my locally cloned version of apel as the + schema file path. + - `mysql -u root validator_db < path_to_apel/schemas/server.sql` + +9. Verify the schema applied correctly, and that the correct tables use a + blackhole engine: + - `mysql -u your_name -p` + - `SHOW DATABASES;` + - `USE validator_db;` + - `SHOW TABLES;` (check all tables are there) + - `SHOW TABLE STATUS;` (check that all tables either have a BLACKHOLE or + NULL engine) + +10. Populate settings.ini with the following, adding in the correct values: + - ``` + [db_validator] + backend=mysql + hostname=localhost + name=validator_db + password= + port=3306 + username= + ``` + - these config options are picked up by the `validatorSettings.py` file.