diff --git a/tools/nv-driver-locator/README.md b/tools/nv-driver-locator/README.md index b250fdf..a795694 100644 --- a/tools/nv-driver-locator/README.md +++ b/tools/nv-driver-locator/README.md @@ -35,232 +35,7 @@ All scripts may be used both as standalone application and importable module. Fo ## Configuration example -```json -{ - "db": { - "type": "file", - "params": { - "workdir": "/var/lib/nv-driver-locator" - } - }, - "key_components": [ - [ - "DriverAttributes", - "Version" - ], - [ - "DriverAttributes", - "Name" - ], - [ - "ChannelAttributes", - "OS" - ] - ], - "channels": [ - { - "type": "gfe_client", - "name": "desktop defaults", - "params": {} - }, - { - "type": "gfe_client", - "name": "desktop beta", - "params": { - "beta": true - } - }, - { - "type": "gfe_client", - "name": "mobile", - "params": { - "notebook": true - } - }, - { - "type": "gfe_client", - "name": "mobile beta", - "params": { - "notebook": true, - "beta": true - } - }, - { - "type": "nvidia_downloads", - "name": "linux beta", - "params": { - "os": "Linux_64", - "product": "GeForce", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "linux stable", - "params": { - "os": "Linux_64", - "product": "GeForce", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win stable", - "params": { - "os": "Windows10_64", - "product": "GeForce", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win beta", - "params": { - "os": "Windows10_64", - "product": "GeForce", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win notebook stable", - "params": { - "os": "Windows10_64", - "product": "GeForceMobile", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win notebook beta", - "params": { - "os": "Windows10_64", - "product": "GeForceMobile", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "linux quadro beta", - "params": { - "os": "Linux_64", - "product": "Quadro", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "linux quadro stable", - "params": { - "os": "Linux_64", - "product": "Quadro", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win quadro stable", - "params": { - "os": "Windows10_64", - "product": "Quadro", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win quadro beta", - "params": { - "os": "Windows10_64", - "product": "Quadro", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win quadro notebook stable", - "params": { - "os": "Windows10_64", - "product": "QuadroMobile", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win quadro notebook beta", - "params": { - "os": "Windows10_64", - "product": "QuadroMobile", - "certlevel": "All" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win server quadro certified", - "params": { - "os": "WindowsServer2012R2_64", - "product": "Quadro", - "certlevel": "Certified" - } - }, - { - "type": "nvidia_downloads", - "name": "downloads win server 2019 quadro certified", - "params": { - "os": "WindowsServer2019", - "product": "Quadro", - "certlevel": "Certified" - } - }, - { - "type": "cuda_downloads", - "name": "cuda toolkit tracker", - "params": {} - }, - { - "type": "vulkan_beta", - "name": "vulkan beta windows", - "params": { - "os": "Windows" - } - }, - { - "type": "vulkan_beta", - "name": "vulkan beta linux", - "params": { - "os": "Linux" - } - } - ], - "notifiers": [ - { - "type": "email", - "name": "my email", - "params": { - "from_addr": "notify-bot@gmail.com", - "to_addrs": [ - "recepient1@domain1.tld", - "recepient2@domain2.tld" - ], - "host": "smtp.google.com", - "use_starttls": true, - "login": "notify-bot", - "password": "MyGoodPass" - } - }, - { - "type": "command", - "name": "sample command", - "params": { - "timeout": 10.0, - "cmdline": [ - "cat", - "-" - ] - } - } - ] -} -``` +See [nv-driver-locator.json.sample](nv-driver-locator.json.sample). ## Components Reference @@ -329,7 +104,19 @@ Type: `vulkan_beta` Params: -* `os` - OS family. Allowed values: `Linux`, `Windows`. Default: `Linux`. +* `timeout` - allowed delay in seconds for each network operation. Default: `10.0` + +#### DebRepoChannel + +Parses Packages or Packages.gz file of deb package repository and extracts versions information. + +Type: `deb_packages` + +Params: + +* `url` - Location of `Packages` or `Packages.gz` file. +* `pkg_pattern` - regexp which defines package name pattern. +* `driver_name` - value returned as `DriverAttributes.Name` for proper aggregation by Hasher. Default: `"Linux x64 (AMD64/EM64T) Display Driver"` * `timeout` - allowed delay in seconds for each network operation. Default: `10.0` ### Notifiers diff --git a/tools/nv-driver-locator/get_deb_drivers.py b/tools/nv-driver-locator/get_deb_drivers.py new file mode 100755 index 0000000..9a1f80f --- /dev/null +++ b/tools/nv-driver-locator/get_deb_drivers.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 + +import urllib.request +import urllib.error +import json +import posixpath +import codecs +import gzip +import sys +from contextlib import contextmanager +import itertools +import string +import codecs +import pprint +import re +import collections + +USER_AGENT = 'Debian APT-HTTP/1.3 (1.6.6)' +DEFAULT_REPO = "https://developer.download.nvidia.com/"\ + "compute/cuda/repos/ubuntu1804/x86_64/Packages.gz" +DEFAULT_REGEX = '^cuda-drivers$' +ENCODING = 'utf-8-sig' + +DriverEntry = collections.namedtuple('DriverEntry', ('name', 'version')) + +def upstream_version(version): + epoch, delim, tail = version.partition(':') + version = tail if delim else epoch + return version.partition('-')[0] + +@contextmanager +def packages_reader(url, timeout): + http_req = urllib.request.Request( + url, + data=None, + headers={ + 'User-Agent': USER_AGENT + } + ) + with urllib.request.urlopen(http_req, None, timeout) as resp: + if url.endswith('.gz') or resp.headers.get('Content-Type', '') == 'application/x-gzip': + with gzip.GzipFile(fileobj=resp) as reader: + yield codecs.getreader(ENCODING)(reader) + else: + yield codecs.getreader(ENCODING)(resp) + +def parse_packages(reader): + for k, g in itertools.groupby(reader, lambda s: bool(s.strip())): + if not k: + continue + pkg = dict() + current_key = None + current_val = None + for line in g: + if line[0] in string.whitespace: + # Continuation + if current_key is None: + continue + current_val += line.rstrip() + else: + # New field + if current_key is not None: + pkg[current_key.lower()] = current_val + current_key, _, current_val = line.partition(':') + current_val = current_val.strip() + if current_key is not None: + pkg[current_key.lower()] = current_val + if 'package' in pkg and 'version' in pkg: + yield pkg + +def _get_deb_versions(*, url=DEFAULT_REPO, name=DEFAULT_REGEX, timeout=10): + pattern = re.compile(name) + with packages_reader(url, timeout) as lines: + for pkg in parse_packages(lines): + if pattern.match(pkg['package']) is not None: + yield DriverEntry(pkg['package'], upstream_version(pkg['version'])) + +def get_deb_versions(*args, **kwargs): + return list(set(_get_deb_versions(*args, **kwargs))) + + +def parse_args(): + import argparse + + def check_positive_float(val): + val = float(val) + if val <= 0: + raise ValueError("Value %s is not valid positive float" % + (repr(val),)) + return val + + parser = argparse.ArgumentParser( + description="Retrieves info about latest NVIDIA drivers available in " + "Nvidia deb packages repositories", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-U", "--url", + default=DEFAULT_REPO, + help="URL for Packages or Packages.gz file") + parser.add_argument("-N", "--name", + default=DEFAULT_REGEX, + help="Package name regexp") + parser.add_argument("-T", "--timeout", + type=check_positive_float, + default=10., + help="timeout for network operations") + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + drv = get_deb_versions(url=args.url, + name=args.name, + timeout=args.timeout) + if drv is None: + print("NOT FOUND") + sys.exit(3) + if False: #not args.raw: + print("Version: %s" % (drv['DriverAttributes']['Version'],)) + print("Beta: %s" % (bool(int(drv['DriverAttributes']['IsBeta'])),)) + print("WHQL: %s" % (bool(int(drv['DriverAttributes']['IsWHQL'])),)) + print("URL: %s" % (drv['DriverAttributes']['DownloadURLAdmin'],)) + else: + json.dump(drv, sys.stdout, indent=4) + sys.stdout.flush() + + +if __name__ == '__main__': + main() diff --git a/tools/nv-driver-locator/nv-driver-locator.json.sample b/tools/nv-driver-locator/nv-driver-locator.json.sample index 9d1061c..ed5514d 100644 --- a/tools/nv-driver-locator/nv-driver-locator.json.sample +++ b/tools/nv-driver-locator/nv-driver-locator.json.sample @@ -242,15 +242,70 @@ { "type": "vulkan_beta", "name": "vulkan beta windows", + "params": {} + }, + { + "type": "deb_packages", + "name": "nvidia cuda drivers deb repo for ubuntu 18.04", "params": { - "os": "Windows" + "url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/Packages.gz", + "pkg_pattern": "^cuda-drivers$" } }, { - "type": "vulkan_beta", - "name": "vulkan beta linux", + "type": "deb_packages", + "name": "nvidia cuda drivers deb repo for ubuntu 20.04", "params": { - "os": "Linux" + "url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/Packages.gz", + "pkg_pattern": "^cuda-drivers$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 18.04 official repos", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/bionic/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 18.04 official repos (updates)", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/bionic-updates/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 18.04 official repos (security)", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/bionic-security/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 20.04 official repos", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/focal/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 20.04 official repos (updates)", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/focal-updates/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" + } + }, + { + "type": "deb_packages", + "name": "ubuntu 20.04 official repos (security)", + "params": { + "url": "http://security.ubuntu.com/ubuntu/dists/focal-security/restricted/binary-amd64/Packages.gz", + "pkg_pattern": "^nvidia-driver-\\d+$" } } ], diff --git a/tools/nv-driver-locator/nv-driver-locator.py b/tools/nv-driver-locator/nv-driver-locator.py index 785ca97..ab67eb1 100755 --- a/tools/nv-driver-locator/nv-driver-locator.py +++ b/tools/nv-driver-locator/nv-driver-locator.py @@ -150,7 +150,7 @@ class CommandNotifier(BaseNotifier): class BaseChannel(ABC): @abstractmethod - def get_latest_driver(self): + def get_latest_drivers(self): pass @@ -175,18 +175,20 @@ class GFEClientChannel(BaseChannel): self._crd = crd self._timeout = timeout gfe_get_driver = importlib.import_module('gfe_get_driver') - self._get_latest_driver = gfe_get_driver.get_latest_geforce_driver + self._get_latest_drivers = gfe_get_driver.get_latest_geforce_driver - def get_latest_driver(self): - res = self._get_latest_driver(notebook=self._notebook, - x86_64=self._x86_64, - os_version=self._os_version, - os_build=self._os_build, - language=self._language, - beta=self._beta, - dch=self._dch, - crd=self._crd, - timeout=self._timeout) + def get_latest_drivers(self): + res = self._get_latest_drivers(notebook=self._notebook, + x86_64=self._x86_64, + os_version=self._os_version, + os_build=self._os_build, + language=self._language, + beta=self._beta, + dch=self._dch, + crd=self._crd, + timeout=self._timeout) + if res is None: + return res.update({ 'ChannelAttributes': { 'Name': self.name, @@ -201,7 +203,7 @@ class GFEClientChannel(BaseChannel): 'Mobile': self._notebook, } }) - return res + yield res class NvidiaDownloadsChannel(BaseChannel): @@ -224,7 +226,7 @@ class NvidiaDownloadsChannel(BaseChannel): self._cuda_ver = gnd.CUDAToolkitVersion[cuda_ver] self._timeout = timeout - def get_latest_driver(self): + def get_latest_drivers(self): latest = self._gnd.get_drivers(os=self._os, product=self._product, certlevel=self._certlevel, @@ -233,7 +235,7 @@ class NvidiaDownloadsChannel(BaseChannel): cuda_ver=self._cuda_ver, timeout=self._timeout) if not latest: - return None + return res = { 'DriverAttributes': { 'Version': latest['version'], @@ -253,7 +255,7 @@ class NvidiaDownloadsChannel(BaseChannel): } if 'download_url' in latest: res['DriverAttributes']['DownloadURL'] = latest['download_url'] - return res + yield res class CudaToolkitDownloadsChannel(BaseChannel): @@ -264,45 +266,81 @@ class CudaToolkitDownloadsChannel(BaseChannel): self._gcd = gcd self._timeout = timeout - def get_latest_driver(self): + def get_latest_drivers(self): latest = self._gcd.get_latest_cuda_tk(timeout=self._timeout) if not latest: - return None - return { + return + yield { 'DriverAttributes': { 'Version': '???', 'Name': latest, 'NameLocalized': latest, + }, + 'ChannelAttributes': { + 'Name': self.name, + 'Type': self.__class__.__name__, } } -@functools.lru_cache(maxsize=0) -def vulkan_downloads(*, timeout=10): - gvd = importlib.import_module('get_vulkan_downloads') - return gvd.get_drivers(timeout=timeout) - class VulkanBetaDownloadsChannel(BaseChannel): def __init__(self, name, *, - os="Linux", timeout=10): self.name = name - self._os = os + self._timeout = timeout + self._gvd = importlib.import_module('get_vulkan_downloads') + + def get_latest_drivers(self): + drivers = self._gvd.get_drivers(timeout=self._timeout) + if drivers is None: + return + for drv in drivers: + yield { + 'DriverAttributes': { + 'Version': drv['version'], + 'Name': drv['name'], + 'NameLocalized': drv['name'], + }, + 'ChannelAttributes': { + 'Name': self.name, + 'Type': self.__class__.__name__, + 'OS': drv['os'], + } + } + +class DebRepoChannel(BaseChannel): + def __init__(self, name, *, + url, + pkg_pattern, + driver_name="Linux x64 (AMD64/EM64T) Display Driver", + timeout=10): + self.name = name + self._gdd = importlib.import_module('get_deb_drivers') + self._url = url + self._pkg_pattern = pkg_pattern + self._driver_name = driver_name self._timeout = timeout - def get_latest_driver(self): - drivers = vulkan_downloads(timeout=self._timeout) + def get_latest_drivers(self): + drivers = self._gdd.get_deb_versions(url=self._url, + name=self._pkg_pattern, + timeout=self._timeout) + if drivers is None: + return for drv in drivers: - if drv["os"] == self._os: - return { - 'DriverAttributes': { - 'Version': drv['version'], - 'Name': drv['name'], - 'NameLocalized': drv['name'], - } + yield { + 'DriverAttributes': { + 'Version': drv.version, + 'DebPkgName': drv.name, + 'Name': self._driver_name, + 'NameLocalized': self._driver_name, + }, + 'ChannelAttributes': { + 'Name': self.name, + 'Type': self.__class__.__name__, + 'OS': 'Linux_64', + 'PkgPattern': self._pkg_pattern, } - else: - return None - + } def parse_args(): parser = argparse.ArgumentParser( @@ -332,6 +370,7 @@ class DriverLocator: 'nvidia_downloads': NvidiaDownloadsChannel, 'cuda_downloads': CudaToolkitDownloadsChannel, 'vulkan_beta': VulkanBetaDownloadsChannel, + 'deb_packages': DebRepoChannel, } channels = [] @@ -389,26 +428,39 @@ class DriverLocator: def run(self): for ch in self._channels: + counter = 0 try: - drv = ch.get_latest_driver() + drivers = ch.get_latest_drivers() except Exception as e: - self._perror("get_latest_driver() invocation failed for " + self._perror("get_latest_drivers() invocation failed for " "channel %s. Exception: %s. Continuing..." % (repr(ch.name), str(e))) continue - if drv is None: - self._perror("Driver not found for channel %s" % - (repr(ch.name),)) - continue + try: - key = self._hasher.hash_object(drv) + # Fetch + for drv in drivers: + counter += 1 + # Hash + try: + key = self._hasher.hash_object(drv) + except Exception as e: + self._perror("Key evaluation failed for channel %s. " + "Exception: %s" % (repr(name), str(e))) + continue + + # Notify + if not self._db.check_key(key): + if self._notify_all(drv): + self._db.set_key(key, drv) except Exception as e: - self._perror("Key evaluation failed for channel %s. " - "Exception: %s" % (repr(name), str(e))) + self._perror("channel %s enumeration terminated with exception: %s" % + (repr(name), str(e))) continue - if not self._db.check_key(key): - if self._notify_all(drv): - self._db.set_key(key, drv) + + if not counter: + self._perror("Drivers not found for channel %s" % + (repr(ch.name),)) return self._ret_code