Mirrored_Repos/nvidia-patch
1
0
Fork 0
mirror of https://github.com/keylase/nvidia-patch.git synced 2024-08-30 18:32:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #338 from Snawoot/linux_nvidia_deb_channel

Browse Source 
Linux nvidia deb channel
This commit is contained in:
Snawoot 2020-10-31 19:42:10 +02:00 committed by GitHub
commit e56416e42a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 304 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
tools/nv-driver-locator/README.md
View File

@ -35,232 +35,7 @@ All scripts may be used both as standalone application and importable module. Fo
## Configuration example
```json
{
"db": {
"type": "file",
"params": {
"workdir": "/var/lib/nv-driver-locator"
}
},
"key_components": [
[
"DriverAttributes",
"Version"
],
[
"DriverAttributes",
"Name"
],
[
"ChannelAttributes",
"OS"
]
],
"channels": [
{
"type": "gfe_client",
"name": "desktop defaults",
"params": {}
},
{
"type": "gfe_client",
"name": "desktop beta",
"params": {
"beta": true
}
},
{
"type": "gfe_client",
"name": "mobile",
"params": {
"notebook": true
}
},
{
"type": "gfe_client",
"name": "mobile beta",
"params": {
"notebook": true,
"beta": true
}
},
{
"type": "nvidia_downloads",
"name": "linux beta",
"params": {
"os": "Linux_64",
"product": "GeForce",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "linux stable",
"params": {
"os": "Linux_64",
"product": "GeForce",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win stable",
"params": {
"os": "Windows10_64",
"product": "GeForce",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win beta",
"params": {
"os": "Windows10_64",
"product": "GeForce",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win notebook stable",
"params": {
"os": "Windows10_64",
"product": "GeForceMobile",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win notebook beta",
"params": {
"os": "Windows10_64",
"product": "GeForceMobile",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "linux quadro beta",
"params": {
"os": "Linux_64",
"product": "Quadro",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "linux quadro stable",
"params": {
"os": "Linux_64",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro stable",
"params": {
"os": "Windows10_64",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro beta",
"params": {
"os": "Windows10_64",
"product": "Quadro",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro notebook stable",
"params": {
"os": "Windows10_64",
"product": "QuadroMobile",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro notebook beta",
"params": {
"os": "Windows10_64",
"product": "QuadroMobile",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win server quadro certified",
"params": {
"os": "WindowsServer2012R2_64",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win server 2019 quadro certified",
"params": {
"os": "WindowsServer2019",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "cuda_downloads",
"name": "cuda toolkit tracker",
"params": {}
},
{
"type": "vulkan_beta",
"name": "vulkan beta windows",
"params": {
"os": "Windows"
}
},
{
"type": "vulkan_beta",
"name": "vulkan beta linux",
"params": {
"os": "Linux"
}
}
],
"notifiers": [
{
"type": "email",
"name": "my email",
"params": {
"from_addr": "notify-bot@gmail.com",
"to_addrs": [
"recepient1@domain1.tld",
"recepient2@domain2.tld"
],
"host": "smtp.google.com",
"use_starttls": true,
"login": "notify-bot",
"password": "MyGoodPass"
}
},
{
"type": "command",
"name": "sample command",
"params": {
"timeout": 10.0,
"cmdline": [
"cat",
"-"
]
}
}
]
}
```
See [nv-driver-locator.json.sample](nv-driver-locator.json.sample).
## Components Reference
@ -329,7 +104,19 @@ Type: `vulkan_beta`
Params:
* `os` - OS family. Allowed values: `Linux`, `Windows`. Default: `Linux`.
* `timeout` - allowed delay in seconds for each network operation. Default: `10.0`
#### DebRepoChannel
Parses Packages or Packages.gz file of deb package repository and extracts versions information.
Type: `deb_packages`
Params:
* `url` - Location of `Packages` or `Packages.gz` file.
* `pkg_pattern` - regexp which defines package name pattern.
* `driver_name` - value returned as `DriverAttributes.Name` for proper aggregation by Hasher. Default: `"Linux x64 (AMD64/EM64T) Display Driver"`
* `timeout` - allowed delay in seconds for each network operation. Default: `10.0`
### Notifiers

129
tools/nv-driver-locator/get_deb_drivers.py Executable file
View File

@ -0,0 +1,129 @@
#!/usr/bin/env python3
import urllib.request
import urllib.error
import json
import posixpath
import codecs
import gzip
import sys
from contextlib import contextmanager
import itertools
import string
import codecs
import pprint
import re
import collections
USER_AGENT = 'Debian APT-HTTP/1.3 (1.6.6)'
DEFAULT_REPO = "https://developer.download.nvidia.com/"\
"compute/cuda/repos/ubuntu1804/x86_64/Packages.gz"
DEFAULT_REGEX = '^cuda-drivers$'
ENCODING = 'utf-8-sig'
DriverEntry = collections.namedtuple('DriverEntry', ('name', 'version'))
def upstream_version(version):
epoch, delim, tail = version.partition(':')
version = tail if delim else epoch
return version.partition('-')[0]
@contextmanager
def packages_reader(url, timeout):
http_req = urllib.request.Request(
url,
data=None,
headers={
'User-Agent': USER_AGENT
}
)
with urllib.request.urlopen(http_req, None, timeout) as resp:
if url.endswith('.gz') or resp.headers.get('Content-Type', '') == 'application/x-gzip':
with gzip.GzipFile(fileobj=resp) as reader:
yield codecs.getreader(ENCODING)(reader)
else:
yield codecs.getreader(ENCODING)(resp)
def parse_packages(reader):
for k, g in itertools.groupby(reader, lambda s: bool(s.strip())):
if not k:
continue
pkg = dict()
current_key = None
current_val = None
for line in g:
if line[0] in string.whitespace:
# Continuation
if current_key is None:
continue
current_val += line.rstrip()
else:
# New field
if current_key is not None:
pkg[current_key.lower()] = current_val
current_key, _, current_val = line.partition(':')
current_val = current_val.strip()
if current_key is not None:
pkg[current_key.lower()] = current_val
if 'package' in pkg and 'version' in pkg:
yield pkg
def _get_deb_versions(*, url=DEFAULT_REPO, name=DEFAULT_REGEX, timeout=10):
pattern = re.compile(name)
with packages_reader(url, timeout) as lines:
for pkg in parse_packages(lines):
if pattern.match(pkg['package']) is not None:
yield DriverEntry(pkg['package'], upstream_version(pkg['version']))
def get_deb_versions(*args, **kwargs):
return list(set(_get_deb_versions(*args, **kwargs)))
def parse_args():
import argparse
def check_positive_float(val):
val = float(val)
if val <= 0:
raise ValueError("Value %s is not valid positive float" %
(repr(val),))
return val
parser = argparse.ArgumentParser(
description="Retrieves info about latest NVIDIA drivers available in "
"Nvidia deb packages repositories",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-U", "--url",
default=DEFAULT_REPO,
help="URL for Packages or Packages.gz file")
parser.add_argument("-N", "--name",
default=DEFAULT_REGEX,
help="Package name regexp")
parser.add_argument("-T", "--timeout",
type=check_positive_float,
default=10.,
help="timeout for network operations")
args = parser.parse_args()
return args
def main():
args = parse_args()
drv = get_deb_versions(url=args.url,
name=args.name,
timeout=args.timeout)
if drv is None:
print("NOT FOUND")
sys.exit(3)
if False: #not args.raw:
print("Version: %s" % (drv['DriverAttributes']['Version'],))
print("Beta: %s" % (bool(int(drv['DriverAttributes']['IsBeta'])),))
print("WHQL: %s" % (bool(int(drv['DriverAttributes']['IsWHQL'])),))
print("URL: %s" % (drv['DriverAttributes']['DownloadURLAdmin'],))
else:
json.dump(drv, sys.stdout, indent=4)
sys.stdout.flush()
if __name__ == '__main__':
main()

63
tools/nv-driver-locator/nv-driver-locator.json.sample
View File

@ -242,15 +242,70 @@
{
"type": "vulkan_beta",
"name": "vulkan beta windows",
"params": {}
},
{
"type": "deb_packages",
"name": "nvidia cuda drivers deb repo for ubuntu 18.04",
"params": {
"os": "Windows"
"url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/Packages.gz",
"pkg_pattern": "^cuda-drivers$"
}
},
{
"type": "vulkan_beta",
"name": "vulkan beta linux",
"type": "deb_packages",
"name": "nvidia cuda drivers deb repo for ubuntu 20.04",
"params": {
"os": "Linux"
"url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/Packages.gz",
"pkg_pattern": "^cuda-drivers$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 18.04 official repos",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/bionic/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 18.04 official repos (updates)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/bionic-updates/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 18.04 official repos (security)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/bionic-security/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 20.04 official repos",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/focal/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 20.04 official repos (updates)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/focal-updates/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 20.04 official repos (security)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/focal-security/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
}
],

112
tools/nv-driver-locator/nv-driver-locator.py
View File

@ -150,7 +150,7 @@ class CommandNotifier(BaseNotifier):
class BaseChannel(ABC):
@abstractmethod
def get_latest_driver(self):
def get_latest_drivers(self):
pass
@ -175,10 +175,10 @@ class GFEClientChannel(BaseChannel):
self._crd = crd
self._timeout = timeout
gfe_get_driver = importlib.import_module('gfe_get_driver')
self._get_latest_driver = gfe_get_driver.get_latest_geforce_driver
self._get_latest_drivers = gfe_get_driver.get_latest_geforce_driver
def get_latest_driver(self):
res = self._get_latest_driver(notebook=self._notebook,
def get_latest_drivers(self):
res = self._get_latest_drivers(notebook=self._notebook,
x86_64=self._x86_64,
os_version=self._os_version,
os_build=self._os_build,
@ -187,6 +187,8 @@ class GFEClientChannel(BaseChannel):
dch=self._dch,
crd=self._crd,
timeout=self._timeout)
if res is None:
return
res.update({
'ChannelAttributes': {
'Name': self.name,
@ -201,7 +203,7 @@ class GFEClientChannel(BaseChannel):
'Mobile': self._notebook,
}
})
return res
yield res
class NvidiaDownloadsChannel(BaseChannel):
@ -224,7 +226,7 @@ class NvidiaDownloadsChannel(BaseChannel):
self._cuda_ver = gnd.CUDAToolkitVersion[cuda_ver]
self._timeout = timeout
def get_latest_driver(self):
def get_latest_drivers(self):
latest = self._gnd.get_drivers(os=self._os,
product=self._product,
certlevel=self._certlevel,
@ -233,7 +235,7 @@ class NvidiaDownloadsChannel(BaseChannel):
cuda_ver=self._cuda_ver,
timeout=self._timeout)
if not latest:
return None
return
res = {
'DriverAttributes': {
'Version': latest['version'],
@ -253,7 +255,7 @@ class NvidiaDownloadsChannel(BaseChannel):
}
if 'download_url' in latest:
res['DriverAttributes']['DownloadURL'] = latest['download_url']
return res
yield res
class CudaToolkitDownloadsChannel(BaseChannel):
@ -264,45 +266,81 @@ class CudaToolkitDownloadsChannel(BaseChannel):
self._gcd = gcd
self._timeout = timeout
def get_latest_driver(self):
def get_latest_drivers(self):
latest = self._gcd.get_latest_cuda_tk(timeout=self._timeout)
if not latest:
return None
return {
return
yield {
'DriverAttributes': {
'Version': '???',
'Name': latest,
'NameLocalized': latest,
},
'ChannelAttributes': {
'Name': self.name,
'Type': self.__class__.__name__,
}
}
@functools.lru_cache(maxsize=0)
def vulkan_downloads(*, timeout=10):
gvd = importlib.import_module('get_vulkan_downloads')
return gvd.get_drivers(timeout=timeout)
class VulkanBetaDownloadsChannel(BaseChannel):
def __init__(self, name, *,
os="Linux",
timeout=10):
self.name = name
self._os = os
self._timeout = timeout
self._gvd = importlib.import_module('get_vulkan_downloads')
def get_latest_driver(self):
drivers = vulkan_downloads(timeout=self._timeout)
def get_latest_drivers(self):
drivers = self._gvd.get_drivers(timeout=self._timeout)
if drivers is None:
return
for drv in drivers:
if drv["os"] == self._os:
return {
yield {
'DriverAttributes': {
'Version': drv['version'],
'Name': drv['name'],
'NameLocalized': drv['name'],
},
'ChannelAttributes': {
'Name': self.name,
'Type': self.__class__.__name__,
'OS': drv['os'],
}
}
else:
return None
class DebRepoChannel(BaseChannel):
def __init__(self, name, *,
url,
pkg_pattern,
driver_name="Linux x64 (AMD64/EM64T) Display Driver",
timeout=10):
self.name = name
self._gdd = importlib.import_module('get_deb_drivers')
self._url = url
self._pkg_pattern = pkg_pattern
self._driver_name = driver_name
self._timeout = timeout
def get_latest_drivers(self):
drivers = self._gdd.get_deb_versions(url=self._url,
name=self._pkg_pattern,
timeout=self._timeout)
if drivers is None:
return
for drv in drivers:
yield {
'DriverAttributes': {
'Version': drv.version,
'DebPkgName': drv.name,
'Name': self._driver_name,
'NameLocalized': self._driver_name,
},
'ChannelAttributes': {
'Name': self.name,
'Type': self.__class__.__name__,
'OS': 'Linux_64',
'PkgPattern': self._pkg_pattern,
}
}
def parse_args():
parser = argparse.ArgumentParser(
@ -332,6 +370,7 @@ class DriverLocator:
'nvidia_downloads': NvidiaDownloadsChannel,
'cuda_downloads': CudaToolkitDownloadsChannel,
'vulkan_beta': VulkanBetaDownloadsChannel,
'deb_packages': DebRepoChannel,
}
channels = []
@ -389,26 +428,39 @@ class DriverLocator:
def run(self):
for ch in self._channels:
counter = 0
try:
drv = ch.get_latest_driver()
drivers = ch.get_latest_drivers()
except Exception as e:
self._perror("get_latest_driver() invocation failed for "
self._perror("get_latest_drivers() invocation failed for "
"channel %s. Exception: %s. Continuing..." %
(repr(ch.name), str(e)))
continue
if drv is None:
self._perror("Driver not found for channel %s" %
(repr(ch.name),))
continue
try:
# Fetch
for drv in drivers:
counter += 1
# Hash
try:
key = self._hasher.hash_object(drv)
except Exception as e:
self._perror("Key evaluation failed for channel %s. "
"Exception: %s" % (repr(name), str(e)))
continue
# Notify
if not self._db.check_key(key):
if self._notify_all(drv):
self._db.set_key(key, drv)
except Exception as e:
self._perror("channel %s enumeration terminated with exception: %s" %
(repr(name), str(e)))
continue
if not counter:
self._perror("Drivers not found for channel %s" %
(repr(ch.name),))
return self._ret_code