Read the data files from the data repository in cxx/data

.gitmodules

@@ -4,9 +4,6 @@
 [submodule "pipeline/owid-python"]
 	path = pipeline/owid-python
 	url = https://github.com/SWAN-community/owid-python.git
-[submodule "ip-intelligence/ip-intelligence-data"]
-	path = ip-intelligence/ip-intelligence-data
-	url = ../ip-intelligence-data
 [submodule "cxx"]
 	path = cxx
 	url = ../51degrees-cxx

AGENTS.md

@@ -39,10 +39,20 @@ so a pipeline change is tested with every product in the same run.
 
 ## Submodules
 
-The C/C++ engines and the data files are submodules, listed in the root
-`.gitmodules` at their paths under each product folder. The IP Intelligence
-on-premise package must compile its own copy of the common C library with
-large data file support; the Device Detection one without it.
+The C/C++ engines are the `cxx` submodule, listed in the root `.gitmodules`.
+The data files come through cxx's own `data` submodule, at
+`cxx/data/device-detection` and `cxx/data/ip-intelligence`, where each free
+file is a `<file>.link` that `cxx/data/get-data-files.ps1` or
+`cxx/data/get-data-files.sh` downloads. Each product's `ci/fetch-assets.ps1`
+gets the files its tests need, and the examples' `example_utils.py` runs the
+script for a file that is missing. The IP Intelligence on-premise package must
+compile its own copy of the common C library with large data file support;
+the Device Detection one without it.
+
+A package's `ExtraPaths` in `ci/packages.json` cannot be a folder inside
+`cxx`, because git reports any change there as a change to `cxx` itself. A
+cxx update therefore releases no package, even when it brings new translations
+in `cxx/data/ip-intelligence`.
 
 ## History
 

README.md

@@ -47,6 +47,29 @@ Clone with the submodules:
 git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git
 ```
 
+The data files the examples and tests use are not stored in a repository. The
+[data](https://github.com/51Degrees/data) repository, which is the `cxx/data`
+submodule, holds a `.link` file for each free one, where the file goes, and
+two scripts that download them. From the root of this repository, run
+
+```
+sh cxx/data/get-data-files.sh
+```
+
+or, in PowerShell,
+
+```
+./cxx/data/get-data-files.ps1
+```
+
+An example whose file is missing runs the script for that file itself.
+
+Git leaves the folder of a removed submodule on disk. A clone made before the
+data files moved to `cxx/data` still has an `ip-intelligence/ip-intelligence-data`
+folder, and keeps `cxx/device-detection-data` and `cxx/ip-intelligence-data`
+once cxx removes them. Delete those folders, so that nothing finds an old copy
+of a data file in them.
+
 Each product's README says what else it needs, such as a C/C++ compiler for
 the on-premise engines and the data files.
 

ci/packages.json

@@ -140,7 +140,6 @@
     "Name": "fiftyone_ipintelligence_onpremise",
     "Product": "ip-intelligence",
     "Path": "ip-intelligence/fiftyone_ipintelligence_onpremise",
-    "ExtraPaths": ["ip-intelligence/ip-intelligence-data"],
     "DependsOn": [
       "fiftyone_ipintelligence_shared",
       "fiftyone_pipeline_engines_fiftyone"
@@ -153,7 +152,6 @@
     "Name": "fiftyone_ipintelligence_translation",
     "Product": "ip-intelligence",
     "Path": "ip-intelligence/fiftyone_ipintelligence_translation",
-    "ExtraPaths": ["ip-intelligence/ip-intelligence-data"],
     "DependsOn": [
       "fiftyone_pipeline_core",
       "fiftyone_pipeline_translation"

device-detection/ci/fetch-assets.ps1

@@ -11,7 +11,8 @@ if ($env:GITHUB_JOB -eq "PreBuild") {
     exit 0
 }
 
-$deviceDetectionData = "$PSScriptRoot/../../cxx/device-detection-data"
+$data = "$PSScriptRoot/../../cxx/data"
+$deviceDetectionData = "$data/device-detection"
 
 $assets = '51Degrees-LiteV4.1.hash', '20000 Evidence Records.yml', '20000 User Agents.csv'
 if ($DeviceDetection) {
@@ -29,3 +30,12 @@ foreach ($asset in $assets) {
         default {New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/$_" -Path "$deviceDetectionData/$_"}
     }
 }
+
+# Downloads any other file the data repository links to in the folder. The
+# script keeps a file that is already there, so the assets linked in above
+# are not downloaded again.
+$global:LASTEXITCODE = 0
+& "$data/get-data-files.ps1" device-detection
+if ($LASTEXITCODE -ne 0) {
+    throw "Not every data file in $deviceDetectionData was downloaded"
+}

device-detection/fiftyone_devicedetection/examples/gettingstarted.py

@@ -24,7 +24,7 @@
 
 # First create the device detection pipeline with the desired settings.
 
-data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash"
+data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash"
 resource_key = "!!YOUR_RESOURCE_KEY!!"
 
 # Uncommment the line below 'On-Premise' and comment the line below 'Cloud' to 

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py

@@ -23,6 +23,8 @@
 from datetime import datetime, timedelta
 from threading import Event, Thread
 import os
+import shlex
+import subprocess
 from pathlib import Path
 
 
@@ -51,6 +53,15 @@ class ExampleUtils:
     # file named in its configuration.
     DATA_FILE_ENV_VAR = "51DEGREES_DD_PATH"
 
+    # The data repository's device detection folder, from the root of a
+    # checkout of this repository. Each file the data repository links to
+    # belongs in this folder, beside its .link file.
+    DATA_FOLDER = os.path.join("cxx", "data", "device-detection")
+
+    # Sent with each download, so the downloads these examples make can be
+    # counted.
+    DOWNLOAD_SOURCE = "examples-51degrees-python"
+
     @staticmethod
     def get_resource_key_from_config(config):
         key = ""
@@ -183,12 +194,29 @@ def __get_env_variable(name):
         else:
             return ""
 
-    # Find the specified filename within the specified directory.
-    # If no directory is specified, the working directory is used.
-    # If the file cannot be found, the algorithm will move to the
-    # parent directory and repeat the process.
+    # Find the specified file. A path to a file that exists is returned as
+    # it is. Next, the data repository's folder is checked in the working
+    # directory and in each directory above it, and a file the data
+    # repository links to is downloaded there when it is missing. Otherwise
+    # the working directory is searched for a file with the same name, then
+    # each parent directory in turn. None is returned when nothing is found.
     @staticmethod
     def find_file(file_name):
+        path = Path(file_name).absolute()
+        if path.is_file():
+            return str(path.resolve())
+
+        # The data repository's copy is preferred to a file with the same
+        # name elsewhere in the tree, which may be out of date or a Git LFS
+        # pointer.
+        current = Path(os.getcwd()).absolute()
+        for root in [current, *current.parents]:
+            candidate = root / ExampleUtils.DATA_FOLDER / path.name
+            if candidate.is_file():
+                return str(candidate)
+            if candidate.with_name(candidate.name + ".link").is_file():
+                return ExampleUtils.__download(str(candidate))
+
         stop_event = Event()
         result = [None]
         # Start the file system search as a separate task.
@@ -235,6 +263,39 @@ def __find_file(file_name, stop_event, result, search_dir=None):
         if search_dir.parent != search_dir and not stop_event.is_set():
             ExampleUtils.__find_file(file_name, stop_event, result, search_dir.parent)
 
+    # Download a file the data repository links to, by running the data
+    # repository's own script for that file, and return its path, or None
+    # when it could not be downloaded. The data repository is the folder
+    # above the one that holds the file's .link.
+    @staticmethod
+    def __download(path):
+        name = os.path.basename(path)
+        data = os.path.dirname(os.path.dirname(path))
+        if os.name == "nt":
+            command = [
+                "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass",
+                "-File", os.path.join(data, "get-data-files.ps1"), name,
+                "-Source", ExampleUtils.DOWNLOAD_SOURCE]
+            text = subprocess.list2cmdline(command)
+        else:
+            command = [
+                "sh", os.path.join(data, "get-data-files.sh"),
+                "-s", ExampleUtils.DOWNLOAD_SOURCE, name]
+            text = shlex.join(command)
+
+        # Flushed first, because the script writes to the same output.
+        print(f"{name} is missing, so it is being downloaded by running\n"
+              f"{text}", flush=True)
+        try:
+            subprocess.run(command, check=False)
+        except OSError:
+            pass
+        if os.path.isfile(path):
+            return path
+        print(f"{name} could not be downloaded. Run this command to try "
+              f"again.\n{text}")
+        return None
+
     @staticmethod
     def get_data_file_date(engine):
         date = engine.engine.getPublishedTime()
@@ -272,9 +333,10 @@ def check_data_file(pipeline, logger):
                 f"that is more than '{ExampleUtils.DATA_FILE_AGE_WARNING}' days " +
                 "old. A more recent data file may be needed to " +
                 "correctly detect the latest devices, browsers, " +
-                "etc. The latest lite data file is available from " +
-                "the device-detection-data repository on GitHub " +
-                "https://github.com/51Degrees/device-detection-data. " +
+                "etc. Get the latest lite data file by running " +
+                "cxx/data/get-data-files.sh -u 51Degrees-LiteV4.1.hash, " +
+                "or cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash " +
+                "-Update on Windows. " +
                 "Find out about the Enterprise data file, which " +
                 "includes automatic updates, on our pricing page: " +
                 "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=data-file-age-warning")

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py

@@ -53,9 +53,8 @@
 #
 # ## Lite Data File
 # Lite data files (free-to-use, limited capabilities, no license key required) are created roughly
-# once a month and cannot be updated using auto-update, they may be downloaded from
-# [Github](https://github.com/51Degrees/device-detection-data) and are included with
-# source distributions of this software.
+# once a month and cannot be updated using auto-update, they may be downloaded with the
+# scripts in the [data](https://github.com/51Degrees/data) repository on GitHub.
 #
 # # Update on Start-Up
 # You can configure the pipeline builder to download an Enterprise data file on start-up.

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py

@@ -152,9 +152,9 @@ def main(argv):
     else:
         logger.log("error",
             "Failed to find a device detection " +
-            "data file. Make sure the device-detection-data " +
-            "submodule has been updated by running " +
-            "`git submodule update --recursive`.")
+            "data file. Make sure the data submodule is there, " +
+            "by running `git submodule update --init --recursive`, " +
+            "and run this example again to download the file.")
 
 if __name__ == "__main__":
     main(sys.argv[1:])

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py

@@ -246,8 +246,9 @@ def build_config():
         if foundDataFile == False:
             raise Exception("Failed to find a device detection data file matching " +
                 f"'{dataFile}'. If using the lite file, then make sure the " +
-                "device-detection-data submodule has been updated by running " +
-                "`git submodule update --recursive`. Otherwise, ensure that the filename " +
+                "data submodule is there, by running " +
+                "`git submodule update --init --recursive`, and run this " +
+                "example again to download the file. Otherwise, ensure that the filename " +
                 "is correct in config.json, or set the environment variable " +
                 f"'{ExampleUtils.DATA_FILE_ENV_VAR}' to the path of the data file.")
 

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html

@@ -30,8 +30,8 @@
         <div class="c-eg-alert">
             WARNING: This example is using a data file that is more than {{ utils.DATA_FILE_AGE_WARNING }}
             days old. A more recent data file may be needed to correctly detect the latest devices,
-            browsers, etc. The latest lite data file is available from the
-            <a href="https://github.com/51Degrees/device-detection-data">device-detection-data</a>
+            browsers, etc. Get the latest lite data file with the scripts in the
+            <a href="https://github.com/51Degrees/data">data</a>
             repository on GitHub. Find out about the Enterprise data file, which includes automatic
             daily updates, on our <a href="https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_web-templates-index.html&utm_term=client-hints">pricing page</a>.
         </div>

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py

@@ -189,8 +189,9 @@ def main(argv):
     else:
         logger.log("error",
             "Failed to find a device detection data file. Make sure the " +
-            "device-detection-data submodule has been updated by running " +
-            "`git submodule update --recursive`.")
+            "data submodule is there, by running " +
+            "`git submodule update --init --recursive`, and run this " +
+            "example again to download the file.")
 
 if __name__ == "__main__":
     main(sys.argv[1:])

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py

@@ -129,8 +129,9 @@ def main(argv):
     else:
         logger.log("error",
             "Failed to find a device detection data file. Make sure the " +
-            "device-detection-data submodule has been updated by running " +
-            "`git submodule update --recursive`.")
+            "data submodule is there, by running " +
+            "`git submodule update --init --recursive`, and run this " +
+            "example again to download the file.")
 
 if __name__ == "__main__":
     main(sys.argv[1:])

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py

@@ -211,8 +211,9 @@ def main(argv):
     else:
         logger.log("error",
             "Failed to find a device detection data file. Make sure the " +
-            "device-detection-data submodule has been updated by running " +
-            "`git submodule update --recursive`.")
+            "data submodule is there, by running " +
+            "`git submodule update --init --recursive`, and run this " +
+            "example again to download the file.")
 
 if __name__ == "__main__":
     main(sys.argv[1:])

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py

@@ -145,7 +145,7 @@ def run(data_file, skip = False):
 if __name__ == "__main__":
     ap = argparse.ArgumentParser(description='Run detection benchmark.')
     ap.add_argument('-d', '--data_file', default='', help='Path to data file')
-    ap.add_argument('-u', '--user_agents_file', default='../../cxx/device-detection-data/20000 User Agents.csv', help='Path to user agents evidence file')
+    ap.add_argument('-u', '--user_agents_file', default='../../cxx/data/device-detection/20000 User Agents.csv', help='Path to user agents evidence file')
     ap.add_argument('-j', '--json_output', default='', help='Output results in JSON format')
     args = ap.parse_args()
     if args.data_file == "":

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py

@@ -184,8 +184,9 @@ def main(argv):
             ShareUsageExample().run(input)
     else:
         print("ERROR: Failed to find the evidence file. Make sure the "
-            "device-detection-data submodule has been updated by running "
-            "`git submodule update --recursive`.")
+            "data submodule is there, by running "
+            "`git submodule update --init --recursive`, and run this "
+            "example again to download the file.")
 
 
 if __name__ == "__main__":

device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py

@@ -148,8 +148,9 @@ def build_config():
         if foundDataFile == False:
             raise Exception("Failed to find a device detection data file matching " +
                 f"'{dataFile}'. If using the lite file, then make sure the " +
-                "device-detection-data submodule has been updated by running " +
-                "`git submodule update --recursive`. Otherwise, ensure that the filename " +
+                "data submodule is there, by running " +
+                "`git submodule update --init --recursive`, and run this " +
+                "example again to download the file. Otherwise, ensure that the filename " +
                 "is correct in config.json.")
 
         return config

device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py

@@ -36,7 +36,7 @@
 from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
 from fiftyone_pipeline_engines.lru_cache import LRUEngineCache
 
-data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash"
+data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash"
 
 mobile_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114"
 
@@ -46,7 +46,7 @@
 
 # User agents file
 
-with open('../../cxx/device-detection-data/20000 User Agents.csv', newline='') as file:
+with open('../../cxx/data/device-detection/20000 User Agents.csv', newline='') as file:
     reader = csv.reader(file)
     user_agents = list(reader)
     no_user_agents = len(user_agents)

device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py

@@ -28,7 +28,7 @@
 from fiftyone_devicedetection_shared.utils import *
 from fiftyone_devicedetection_onpremise import constants
 
-data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash"
+data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash"
 mobile_ua = ("Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) "
             "AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile"
             "/11D167 Safari/9537.53")

device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py

@@ -28,7 +28,7 @@
 from fiftyone_devicedetection_onpremise.swig_data import SwigData
 from fiftyone_devicedetection_shared.utils import *
 
-data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash"
+data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash"
 
 # Create a simple pipeline to access the engine with and process it with flow data
 deviceDetectionOnPremiseEngine = DeviceDetectionOnPremise(