Mirrored_Repos/nvidia-patch
1
0
Fork 0
mirror of https://github.com/keylase/nvidia-patch.git synced 2024-08-30 18:32:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #338 from Snawoot/linux_nvidia_deb_channel

Browse Source 
Linux nvidia deb channel
This commit is contained in:
Snawoot 2020-10-31 19:42:10 +02:00 committed by GitHub
commit e56416e42a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 304 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
tools/nv-driver-locator/README.md
View File

@ -35,232 +35,7 @@ All scripts may be used both as standalone application and importable module. Fo
## Configuration example ## Configuration example
```json See [nv-driver-locator.json.sample](nv-driver-locator.json.sample).
{
"db": {
"type": "file",
"params": {
"workdir": "/var/lib/nv-driver-locator"
}
},
"key_components": [
[
"DriverAttributes",
"Version"
],
[
"DriverAttributes",
"Name"
],
[
"ChannelAttributes",
"OS"
]
],
"channels": [
{
"type": "gfe_client",
"name": "desktop defaults",
"params": {}
},
{
"type": "gfe_client",
"name": "desktop beta",
"params": {
"beta": true
}
},
{
"type": "gfe_client",
"name": "mobile",
"params": {
"notebook": true
}
},
{
"type": "gfe_client",
"name": "mobile beta",
"params": {
"notebook": true,
"beta": true
}
},
{
"type": "nvidia_downloads",
"name": "linux beta",
"params": {
"os": "Linux_64",
"product": "GeForce",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "linux stable",
"params": {
"os": "Linux_64",
"product": "GeForce",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win stable",
"params": {
"os": "Windows10_64",
"product": "GeForce",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win beta",
"params": {
"os": "Windows10_64",
"product": "GeForce",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win notebook stable",
"params": {
"os": "Windows10_64",
"product": "GeForceMobile",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win notebook beta",
"params": {
"os": "Windows10_64",
"product": "GeForceMobile",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "linux quadro beta",
"params": {
"os": "Linux_64",
"product": "Quadro",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "linux quadro stable",
"params": {
"os": "Linux_64",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro stable",
"params": {
"os": "Windows10_64",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro beta",
"params": {
"os": "Windows10_64",
"product": "Quadro",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro notebook stable",
"params": {
"os": "Windows10_64",
"product": "QuadroMobile",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win quadro notebook beta",
"params": {
"os": "Windows10_64",
"product": "QuadroMobile",
"certlevel": "All"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win server quadro certified",
"params": {
"os": "WindowsServer2012R2_64",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "nvidia_downloads",
"name": "downloads win server 2019 quadro certified",
"params": {
"os": "WindowsServer2019",
"product": "Quadro",
"certlevel": "Certified"
}
},
{
"type": "cuda_downloads",
"name": "cuda toolkit tracker",
"params": {}
},
{
"type": "vulkan_beta",
"name": "vulkan beta windows",
"params": {
"os": "Windows"
}
},
{
"type": "vulkan_beta",
"name": "vulkan beta linux",
"params": {
"os": "Linux"
}
}
],
"notifiers": [
{
"type": "email",
"name": "my email",
"params": {
"from_addr": "notify-bot@gmail.com",
"to_addrs": [
"recepient1@domain1.tld",
"recepient2@domain2.tld"
],
"host": "smtp.google.com",
"use_starttls": true,
"login": "notify-bot",
"password": "MyGoodPass"
}
},
{
"type": "command",
"name": "sample command",
"params": {
"timeout": 10.0,
"cmdline": [
"cat",
"-"
]
}
}
]
}
```
## Components Reference ## Components Reference
@ -329,7 +104,19 @@ Type: `vulkan_beta`
Params: Params:
* `os` - OS family. Allowed values: `Linux`, `Windows`. Default: `Linux`. * `timeout` - allowed delay in seconds for each network operation. Default: `10.0`
#### DebRepoChannel
Parses Packages or Packages.gz file of deb package repository and extracts versions information.
Type: `deb_packages`
Params:
* `url` - Location of `Packages` or `Packages.gz` file.
* `pkg_pattern` - regexp which defines package name pattern.
* `driver_name` - value returned as `DriverAttributes.Name` for proper aggregation by Hasher. Default: `"Linux x64 (AMD64/EM64T) Display Driver"`
* `timeout` - allowed delay in seconds for each network operation. Default: `10.0` * `timeout` - allowed delay in seconds for each network operation. Default: `10.0`
### Notifiers ### Notifiers

129
tools/nv-driver-locator/get_deb_drivers.py Executable file
View File

@ -0,0 +1,129 @@
#!/usr/bin/env python3
import urllib.request
import urllib.error
import json
import posixpath
import codecs
import gzip
import sys
from contextlib import contextmanager
import itertools
import string
import codecs
import pprint
import re
import collections
USER_AGENT = 'Debian APT-HTTP/1.3 (1.6.6)'
DEFAULT_REPO = "https://developer.download.nvidia.com/"\
"compute/cuda/repos/ubuntu1804/x86_64/Packages.gz"
DEFAULT_REGEX = '^cuda-drivers$'
ENCODING = 'utf-8-sig'
DriverEntry = collections.namedtuple('DriverEntry', ('name', 'version'))
def upstream_version(version):
epoch, delim, tail = version.partition(':')
version = tail if delim else epoch
return version.partition('-')[0]
@contextmanager
def packages_reader(url, timeout):
http_req = urllib.request.Request(
url,
data=None,
headers={
'User-Agent': USER_AGENT
}
)
with urllib.request.urlopen(http_req, None, timeout) as resp:
if url.endswith('.gz') or resp.headers.get('Content-Type', '') == 'application/x-gzip':
with gzip.GzipFile(fileobj=resp) as reader:
yield codecs.getreader(ENCODING)(reader)
else:
yield codecs.getreader(ENCODING)(resp)
def parse_packages(reader):
for k, g in itertools.groupby(reader, lambda s: bool(s.strip())):
if not k:
continue
pkg = dict()
current_key = None
current_val = None
for line in g:
if line[0] in string.whitespace:
# Continuation
if current_key is None:
continue
current_val += line.rstrip()
else:
# New field
if current_key is not None:
pkg[current_key.lower()] = current_val
current_key, _, current_val = line.partition(':')
current_val = current_val.strip()
if current_key is not None:
pkg[current_key.lower()] = current_val
if 'package' in pkg and 'version' in pkg:
yield pkg
def _get_deb_versions(*, url=DEFAULT_REPO, name=DEFAULT_REGEX, timeout=10):
pattern = re.compile(name)
with packages_reader(url, timeout) as lines:
for pkg in parse_packages(lines):
if pattern.match(pkg['package']) is not None:
yield DriverEntry(pkg['package'], upstream_version(pkg['version']))
def get_deb_versions(*args, **kwargs):
return list(set(_get_deb_versions(*args, **kwargs)))
def parse_args():
import argparse
def check_positive_float(val):
val = float(val)
if val <= 0:
raise ValueError("Value %s is not valid positive float" %
(repr(val),))
return val
parser = argparse.ArgumentParser(
description="Retrieves info about latest NVIDIA drivers available in "
"Nvidia deb packages repositories",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-U", "--url",
default=DEFAULT_REPO,
help="URL for Packages or Packages.gz file")
parser.add_argument("-N", "--name",
default=DEFAULT_REGEX,
help="Package name regexp")
parser.add_argument("-T", "--timeout",
type=check_positive_float,
default=10.,
help="timeout for network operations")
args = parser.parse_args()
return args
def main():
args = parse_args()
drv = get_deb_versions(url=args.url,
name=args.name,
timeout=args.timeout)
if drv is None:
print("NOT FOUND")
sys.exit(3)
if False: #not args.raw:
print("Version: %s" % (drv['DriverAttributes']['Version'],))
print("Beta: %s" % (bool(int(drv['DriverAttributes']['IsBeta'])),))
print("WHQL: %s" % (bool(int(drv['DriverAttributes']['IsWHQL'])),))
print("URL: %s" % (drv['DriverAttributes']['DownloadURLAdmin'],))
else:
json.dump(drv, sys.stdout, indent=4)
sys.stdout.flush()
if __name__ == '__main__':
main()

63
tools/nv-driver-locator/nv-driver-locator.json.sample
View File

@ -242,15 +242,70 @@
{ {
"type": "vulkan_beta", "type": "vulkan_beta",
"name": "vulkan beta windows", "name": "vulkan beta windows",
"params": {}
},
{
"type": "deb_packages",
"name": "nvidia cuda drivers deb repo for ubuntu 18.04",
"params": { "params": {
"os": "Windows" "url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/Packages.gz",
"pkg_pattern": "^cuda-drivers$"
} }
}, },
{ {
"type": "vulkan_beta", "type": "deb_packages",
"name": "vulkan beta linux", "name": "nvidia cuda drivers deb repo for ubuntu 20.04",
"params": { "params": {
"os": "Linux" "url": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/Packages.gz",
"pkg_pattern": "^cuda-drivers$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 18.04 official repos",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/bionic/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 18.04 official repos (updates)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/bionic-updates/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 18.04 official repos (security)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/bionic-security/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 20.04 official repos",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/focal/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 20.04 official repos (updates)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/focal-updates/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
}
},
{
"type": "deb_packages",
"name": "ubuntu 20.04 official repos (security)",
"params": {
"url": "http://security.ubuntu.com/ubuntu/dists/focal-security/restricted/binary-amd64/Packages.gz",
"pkg_pattern": "^nvidia-driver-\\d+$"
} }
} }
], ],

152
tools/nv-driver-locator/nv-driver-locator.py
View File

@ -150,7 +150,7 @@ class CommandNotifier(BaseNotifier):
class BaseChannel(ABC): class BaseChannel(ABC):
@abstractmethod @abstractmethod
def get_latest_driver(self): def get_latest_drivers(self):
pass pass
@ -175,18 +175,20 @@ class GFEClientChannel(BaseChannel):
self._crd = crd self._crd = crd
self._timeout = timeout self._timeout = timeout
gfe_get_driver = importlib.import_module('gfe_get_driver') gfe_get_driver = importlib.import_module('gfe_get_driver')
self._get_latest_driver = gfe_get_driver.get_latest_geforce_driver self._get_latest_drivers = gfe_get_driver.get_latest_geforce_driver
def get_latest_driver(self): def get_latest_drivers(self):
res = self._get_latest_driver(notebook=self._notebook, res = self._get_latest_drivers(notebook=self._notebook,
x86_64=self._x86_64, x86_64=self._x86_64,
os_version=self._os_version, os_version=self._os_version,
os_build=self._os_build, os_build=self._os_build,
language=self._language, language=self._language,
beta=self._beta, beta=self._beta,
dch=self._dch, dch=self._dch,
crd=self._crd, crd=self._crd,
timeout=self._timeout) timeout=self._timeout)
if res is None:
return
res.update({ res.update({
'ChannelAttributes': { 'ChannelAttributes': {
'Name': self.name, 'Name': self.name,
@ -201,7 +203,7 @@ class GFEClientChannel(BaseChannel):
'Mobile': self._notebook, 'Mobile': self._notebook,
} }
}) })
return res yield res
class NvidiaDownloadsChannel(BaseChannel): class NvidiaDownloadsChannel(BaseChannel):
@ -224,7 +226,7 @@ class NvidiaDownloadsChannel(BaseChannel):
self._cuda_ver = gnd.CUDAToolkitVersion[cuda_ver] self._cuda_ver = gnd.CUDAToolkitVersion[cuda_ver]
self._timeout = timeout self._timeout = timeout
def get_latest_driver(self): def get_latest_drivers(self):
latest = self._gnd.get_drivers(os=self._os, latest = self._gnd.get_drivers(os=self._os,
product=self._product, product=self._product,
certlevel=self._certlevel, certlevel=self._certlevel,
@ -233,7 +235,7 @@ class NvidiaDownloadsChannel(BaseChannel):
cuda_ver=self._cuda_ver, cuda_ver=self._cuda_ver,
timeout=self._timeout) timeout=self._timeout)
if not latest: if not latest:
return None return
res = { res = {
'DriverAttributes': { 'DriverAttributes': {
'Version': latest['version'], 'Version': latest['version'],
@ -253,7 +255,7 @@ class NvidiaDownloadsChannel(BaseChannel):
} }
if 'download_url' in latest: if 'download_url' in latest:
res['DriverAttributes']['DownloadURL'] = latest['download_url'] res['DriverAttributes']['DownloadURL'] = latest['download_url']
return res yield res
class CudaToolkitDownloadsChannel(BaseChannel): class CudaToolkitDownloadsChannel(BaseChannel):
@ -264,45 +266,81 @@ class CudaToolkitDownloadsChannel(BaseChannel):
self._gcd = gcd self._gcd = gcd
self._timeout = timeout self._timeout = timeout
def get_latest_driver(self): def get_latest_drivers(self):
latest = self._gcd.get_latest_cuda_tk(timeout=self._timeout) latest = self._gcd.get_latest_cuda_tk(timeout=self._timeout)
if not latest: if not latest:
return None return
return { yield {
'DriverAttributes': { 'DriverAttributes': {
'Version': '???', 'Version': '???',
'Name': latest, 'Name': latest,
'NameLocalized': latest, 'NameLocalized': latest,
},
'ChannelAttributes': {
'Name': self.name,
'Type': self.__class__.__name__,
} }
} }
@functools.lru_cache(maxsize=0)
def vulkan_downloads(*, timeout=10):
gvd = importlib.import_module('get_vulkan_downloads')
return gvd.get_drivers(timeout=timeout)
class VulkanBetaDownloadsChannel(BaseChannel): class VulkanBetaDownloadsChannel(BaseChannel):
def __init__(self, name, *, def __init__(self, name, *,
os="Linux",
timeout=10): timeout=10):
self.name = name self.name = name
self._os = os self._timeout = timeout
self._gvd = importlib.import_module('get_vulkan_downloads')
def get_latest_drivers(self):
drivers = self._gvd.get_drivers(timeout=self._timeout)
if drivers is None:
return
for drv in drivers:
yield {
'DriverAttributes': {
'Version': drv['version'],
'Name': drv['name'],
'NameLocalized': drv['name'],
},
'ChannelAttributes': {
'Name': self.name,
'Type': self.__class__.__name__,
'OS': drv['os'],
}
}
class DebRepoChannel(BaseChannel):
def __init__(self, name, *,
url,
pkg_pattern,
driver_name="Linux x64 (AMD64/EM64T) Display Driver",
timeout=10):
self.name = name
self._gdd = importlib.import_module('get_deb_drivers')
self._url = url
self._pkg_pattern = pkg_pattern
self._driver_name = driver_name
self._timeout = timeout self._timeout = timeout
def get_latest_driver(self): def get_latest_drivers(self):
drivers = vulkan_downloads(timeout=self._timeout) drivers = self._gdd.get_deb_versions(url=self._url,
name=self._pkg_pattern,
timeout=self._timeout)
if drivers is None:
return
for drv in drivers: for drv in drivers:
if drv["os"] == self._os: yield {
return { 'DriverAttributes': {
'DriverAttributes': { 'Version': drv.version,
'Version': drv['version'], 'DebPkgName': drv.name,
'Name': drv['name'], 'Name': self._driver_name,
'NameLocalized': drv['name'], 'NameLocalized': self._driver_name,
} },
'ChannelAttributes': {
'Name': self.name,
'Type': self.__class__.__name__,
'OS': 'Linux_64',
'PkgPattern': self._pkg_pattern,
} }
else: }
return None
def parse_args(): def parse_args():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
@ -332,6 +370,7 @@ class DriverLocator:
'nvidia_downloads': NvidiaDownloadsChannel, 'nvidia_downloads': NvidiaDownloadsChannel,
'cuda_downloads': CudaToolkitDownloadsChannel, 'cuda_downloads': CudaToolkitDownloadsChannel,
'vulkan_beta': VulkanBetaDownloadsChannel, 'vulkan_beta': VulkanBetaDownloadsChannel,
'deb_packages': DebRepoChannel,
} }
channels = [] channels = []
@ -389,26 +428,39 @@ class DriverLocator:
def run(self): def run(self):
for ch in self._channels: for ch in self._channels:
counter = 0
try: try:
drv = ch.get_latest_driver() drivers = ch.get_latest_drivers()
except Exception as e: except Exception as e:
self._perror("get_latest_driver() invocation failed for " self._perror("get_latest_drivers() invocation failed for "
"channel %s. Exception: %s. Continuing..." % "channel %s. Exception: %s. Continuing..." %
(repr(ch.name), str(e))) (repr(ch.name), str(e)))
continue continue
if drv is None:
self._perror("Driver not found for channel %s" %
(repr(ch.name),))
continue
try: try:
key = self._hasher.hash_object(drv) # Fetch
for drv in drivers:
counter += 1
# Hash
try:
key = self._hasher.hash_object(drv)
except Exception as e:
self._perror("Key evaluation failed for channel %s. "
"Exception: %s" % (repr(name), str(e)))
continue
# Notify
if not self._db.check_key(key):
if self._notify_all(drv):
self._db.set_key(key, drv)
except Exception as e: except Exception as e:
self._perror("Key evaluation failed for channel %s. " self._perror("channel %s enumeration terminated with exception: %s" %
"Exception: %s" % (repr(name), str(e))) (repr(name), str(e)))
continue continue
if not self._db.check_key(key):
if self._notify_all(drv): if not counter:
self._db.set_key(key, drv) self._perror("Drivers not found for channel %s" %
(repr(ch.name),))
return self._ret_code return self._ret_code