Current File : //lib/python3/dist-packages/softwareproperties/shortcuthandler.py
# Copyright (c) 2019 Canonical Ltd.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA
import apt_pkg
import os
import re
import subprocess
import tempfile
from aptsources.distro import get_distro
from aptsources.sourceslist import (SourceEntry, SourcesList, Deb822SourceEntry)
from contextlib import suppress
from gettext import gettext as _
from urllib.parse import urlparse
from softwareproperties.sourceutils import (get_source_entry_from_list,
copy_source_entry,
deb822_source_entry_contains)
apt_pkg.init()
GPG_KEYRING_CMD = 'gpg -q --no-options --no-default-keyring --batch --keyring %s'
class ShortcutHandler(object):
'''Superclass for shortcut handler implementations.
This provides a way to take a apt repository reference, in various forms,
and write the specific apt configuration to local files. This also can
remove previously written configuration from local files.
This class and any subclasses should never modify any main apt configuration
files, only specifically named files in '.d' subdirs (e.g. sources.list.d, etc)
should be modified. The only exception to that rule is adding or removing
sourceslist lines or components of existing source entries.
'''
def __init__(self, shortcut, components=None, enable_source=False,
codename=None, pocket=None, dry_run=False, deb822=False, **kwargs):
self.shortcut = shortcut
self.components = components or []
self.enable_source = enable_source
self.distro = get_distro()
self.codename = codename or self.distro.codename
self.pocket = pocket
self.dry_run = dry_run
self.deb822 = deb822
# Subclasses should not directly reference _source_entry,
# use _set_source_entry() and SourceEntry()
self._source_entry = None
# Subclasses should directly set these fields, if appropriate
self._filebase = None
self._username = None
self._password = None
@classmethod
def is_valid_uri(cls, uri):
'''Return if the uri is in valid uri format'''
parsed = urlparse(uri)
return parsed.scheme and parsed.netloc
@classmethod
def uri_strip_auth(cls, uri):
'''Return the uri with the username and password stripped'''
parsed = urlparse(uri)
# urlparse doesn't have any great way to simply remove the auth data,
# so let's just strip everything to the left of '@'
return parsed._replace(netloc=parsed.netloc.rpartition('@')[2]).geturl()
@classmethod
def uri_insert_auth(cls, uri, username, password):
'''Return the uri with the username and password included'''
parsed = urlparse(cls.uri_strip_auth(uri))
netloc='%s:%s@%s' % (username, password, parsed.netloc)
return parsed._replace(netloc=netloc).geturl()
@classmethod
def fingerprints(cls, keys):
'''Return an array of fingerprint(s) for provided key(s).
The 'keys' parameter should be in text (str) or binary (bytes) format;
it is converted to bytes if needed, and then passed to the 'gpg' program.
'''
cmd = 'gpg -q --no-options --no-keyring --batch --with-colons'
# yes, --with-fingerprint twice, to print subkey fingerprints
cmd += ' --with-fingerprint' * 2
try:
with tempfile.TemporaryDirectory() as homedir:
cmd += f' --homedir {homedir}'
if not isinstance(keys, bytes):
keys = keys.encode()
stdout = subprocess.run(cmd.split(), check=True, input=keys,
stdout=subprocess.PIPE).stdout.decode()
except subprocess.CalledProcessError as e:
print(_("Warning: gpg error while processing keys:\n%s") % e)
return []
try:
# gpg --with-colons fpr field puts fingerprint into (1-based) field 10
return [l.split(':')[9] for l in stdout.splitlines() if l.startswith('fpr')]
except KeyError:
print(_("Warning: invalid gpg output:\n%s") % stdout)
return []
@property
def description(self):
return (_("Archive for codename: %s components: %s" %
(self.SourceEntry().dist,
','.join(self.SourceEntry().comps))))
@property
def web_link(self):
return self.archive_link
@property
def archive_link(self):
return self.SourceEntry().uri
@property
def dist(self):
if self.pocket:
return '%s-%s' % (self.codename, self.pocket)
return self.codename
@property
def binary_type(self):
'''Text indicating a binary-type SourceEntry.'''
return self.distro.binary_type
@property
def source_type(self):
'''Text indicating a source-type SourceEntry.'''
return self.distro.source_type
def SourceEntry(self, pkgtype=None):
'''Get the SourceEntry representing this archive/shortcut.
This should never include any authentication data; if required,
the username and password should only be available from the
username and password properties, as well as from the
netrcparts_content property.
If pkgtype is provided, it must be either binary_type or source_type,
in which case this returns a SourceEntry with the requested type.
If pkgtype is not specified, this returns a SourceEntry with an
implementation-dependent type (in most cases, implementations should
default to binary_type).
Note that the default SourceEntry will be returned without modification,
and the implementation will determine if it is enabled or disabled;
while the source-type SourceEntry will be enabled or disabled based on
self.enable_source. The binary-type SourceEntry will always be enabled.
The SourceEntry 'file' field should always be set to the value of
sourceparts_file.
'''
if not self._source_entry:
raise NotImplementedError('Implementation class did not set self._source_entry')
e = copy_source_entry(self._source_entry)
if not pkgtype:
return e
if pkgtype == self.binary_type:
e.set_enabled(True)
elif pkgtype == self.source_type:
e.set_enabled(self.enable_source)
else:
raise ValueError('Invalid pkgtype: %s' % pkgtype)
if isinstance(e, Deb822SourceEntry):
e.types = [pkgtype]
else:
e.type = pkgtype
return copy_source_entry(e)
@property
def username(self):
'''Return the username used for authentication
If authentication is used, return the username; otherwise return None.
By default, this returns the private variable self._username, which
defaults to None. Subclasses should override this method and/or
set self._username if they have authentication data.
'''
return self._username
@property
def password(self):
'''Return the password used for authentication
If authentication is used, return the password; otherwise return None.
By default, this returns the private variable self._password, which
defaults to None. Subclasses should override this method and/or
set self._password if they have authentication data.
'''
return self._password
def add(self):
'''Save all data for this shortcut to file(s).
This writes everything to the relevant files. By default, it
calls add_source(), add_key(), and add_login(). Subclasses
should override it if other actions are required.
'''
self.add_source()
self.add_key()
self.add_login()
def remove(self):
'''Remove all data for this shortcut from file(s).
This removes everything from the relevant files. By default, it
only calls remove_source() and remove_login(). Subclasses
should override it if other actions are required. Note that by
default is does not call remove_key().
'''
self.remove_source()
self.remove_login()
def _add_deb822_source(self):
sourceslist = SourcesList(deb822=True)
newentry = None
binentry = self.SourceEntry(self.binary_type)
for s in sourceslist:
if not isinstance(s, Deb822SourceEntry):
continue
if deb822_source_entry_contains(s, binentry):
newentry = s
break
if newentry is not None:
print(_("Found existing %s entry in %s") % (self.binary_type,
newentry.file))
else:
newentry = sourceslist.add('', '', '', [],
file=self.sourceparts_file)
newentry.types = binentry.types
newentry.uris = binentry.uris
newentry.suites = binentry.suites
newentry.comps = binentry.comps
if self.trustedparts_content:
lines = self.trustedparts_content.splitlines()
lines = [' ' + (l if l.strip() else '.') for l in lines]
newentry.section['Signed-By'] = '\n' + '\n'.join(lines)
binentry = newentry
if self.enable_source and self.source_type not in set(binentry.types):
newentry = None
srcentry = self.SourceEntry(self.source_type)
for s in sourceslist:
if not isinstance(s, Deb822SourceEntry):
continue
if deb822_source_entry_contains(s, srcentry):
newentry = s
break
if newentry is not None:
print(_("Found existing %s entry in %s") % (self.source_type,
newentry.file))
else:
binentry.types = [self.binary_type, self.source_type]
if not self.dry_run:
if not os.path.exists(binentry.file):
# Create the dir if needed
if (binentry.file.startswith(self.sourceparts_path)
and not os.path.exists(self.sourceparts_path)):
os.mkdir(self.sourceparts_path, 0o755)
with open(binentry.file, 'w'):
os.chmod(binentry.file, self.sourceparts_mode)
sourceslist.save()
def add_source(self):
'''Add the apt SourceEntries.
This uses SourcesList to add the binary-type and source-type
SourceEntries.
If the SourceEntry matches a known apt template, this will ignore
the sourceparts_file and instead place the SourceEntries into
the main/default sources.list file. Otherwise, this will add
the SourceEntries into the sourceparts_file.
If either the binary-type or source-type entry exist in the current
SourcesList, the existing entries are updated instead of placing
the entries in the sourceparts_file.
'''
if self.deb822:
self._add_deb822_source()
return
binentry = self.SourceEntry(self.binary_type)
srcentry = self.SourceEntry(self.source_type)
mode = self.sourceparts_mode
sourceslist = SourcesList()
newentry = get_source_entry_from_list(sourceslist, binentry)
if newentry is not None:
print(_("Found existing %s entry in %s") % (newentry.type, newentry.file))
else:
newentry = sourceslist.add(
uri=binentry.uri,
dist=binentry.dist,
file=binentry.file,
type=binentry.type,
orig_comps=binentry.comps
)
newentry.disabled = binentry.disabled
if binentry.file != newentry.file:
# existing binentry, but not in file we were expecting, just update it
print(_("Updating existing entry instead of using %s") % binentry.file)
elif newentry.template:
# our SourceEntry matches a template; use default sources.list file
newentry.file = SourceEntry('').file
print(_("Archive has template, updating %s") % newentry.file)
elif binentry.disabled:
print(_("Adding disabled %s entry to %s") % (newentry.type, newentry.file))
else:
print(_("Adding %s entry to %s") % (newentry.type, newentry.file))
binentry = newentry
# Unless it already exists somewhere, add the srcentry right after the binentry
srcentry.file = binentry.file
newentry = get_source_entry_from_list(sourceslist, srcentry)
if newentry is not None:
print(_("Found existing %s entry in %s") % (newentry.type, newentry.file))
else:
newentry = sourceslist.add(
uri=srcentry.uri,
dist=srcentry.dist,
file=srcentry.file,
type=srcentry.type,
orig_comps=srcentry.comps
)
newentry.disabled = srcentry.disabled
if srcentry.file != newentry.file:
# existing srcentry, but not in file we were expecting, just update it
print(_("Updating existing entry instead of using %s") % srcentry.file)
elif srcentry.disabled:
print(_("Adding disabled %s entry to %s") % (newentry.type, newentry.file))
else:
print(_("Adding %s entry to %s") % (newentry.type, newentry.file))
srcentry = newentry
if not self.dry_run:
# If the file doesn't exist, create it so we can set the mode
for entryfile in set([binentry.file, srcentry.file]):
if not os.path.exists(entryfile):
# Create the dir if needed
if (entryfile.startswith(self.sourceparts_path)
and not os.path.exists(self.sourceparts_path)):
os.mkdir(self.sourceparts_path, 0o755)
with open(entryfile, 'w'):
os.chmod(entryfile, mode)
sourceslist.save()
def _remove_deb822_source(self):
sourceslist = SourcesList(deb822=True)
binentry = self.SourceEntry(self.binary_type)
for s in sourceslist:
if not isinstance(s, Deb822SourceEntry):
continue
if deb822_source_entry_contains(s, binentry):
print(_("Removing entry from %s") % (s.file))
binentry = s
sourceslist.remove(binentry)
break
if not self.dry_run:
sourceslist.save()
if not [s for s in sourceslist if s.file == binentry.file]:
try:
os.remove(binentry.file)
except (OSError, IOError):
pass
def remove_source(self):
'''Remove the apt SourceEntries.
This uses SourcesList to remove the binary-type and source-type
SourceEntries.
This must disable the corresponding SourceEntries, from whatever file(s)
they are located in. This must not disable more than matches, e.g.
if the existing SourceEntry line contains more components this must
edit the existing line to remove this SourceEntry's component(s).
After disabling all matching SourceEntries, if the sourceparts_file is
empty or contains only invalid and/or disabled SourceEntries, this
may remove the sourceparts_file.
'''
if self.deb822:
self._remove_deb822_source()
return
sourceslist = SourcesList()
orig_files = set([e.file for e in sourceslist])
binentry = self.SourceEntry(self.binary_type)
srcentry = self.SourceEntry(self.source_type)
binentry.disabled = False
srcentry.disabled = False
for s in sourceslist:
if get_source_entry_from_list([binentry,srcentry], s) is not None:
print(_("Disabling %s entry in %s") % (s.type, s.file))
s.disabled = True
file_entries = [s for s in sourceslist if s.file == self.sourceparts_file]
if not [e for e in file_entries if not e.invalid and not e.disabled]:
# no more valid/enabled entries in our file, remove them
for e in file_entries:
if not e.invalid:
print(_("Removing disabled %s entry from %s") % (e.type, e.file))
sourceslist.remove(e)
if not self.dry_run:
sourceslist.save()
for file in orig_files - set([e.file for e in sourceslist]):
try:
os.remove(file)
except (OSError, IOError):
pass
@property
def sourceparts_path(self):
'''Return result of apt_pkg.config.find_dir("Dir::Etc::sourceparts")'''
return apt_pkg.config.find_dir("Dir::Etc::sourceparts")
@property
def sourceparts_filename(self):
'''Get the sources.list.d filename, without the leading path.
By default, this combines the filebase with the codename, and uses a
extension of 'list' if deb822 is not enabled, and 'sources' if deb822
is enabled. This is different than the trustedparts or
netrcparts filenames, which use only the filebase plus extension.
'''
ext = 'sources' if self.deb822 else 'list'
return self._filebase_to_filename(ext, suffix=self.codename)
@property
def sourceparts_file(self):
'''Get the sources.list.d absolute-path filename.
Note that the add_source() function will not use this file if this shortcut's
SourceEntry matches a known apt template; instead the entries will be placed
in the main sources.list file. Also, if the SourceEntry already exists in
the SourcesList, it will be edited in place, instead of using this file.
See add_source() for more details.
'''
return self._filename_to_file(self.sourceparts_path, self.sourceparts_filename)
@property
def sourceparts_mode(self):
'''Mode of sourceparts file.
Note that add_source() will only use this mode if it creates a new file
for sourceparts_file; if the file already exists or if the SourceEntry is
saved in a different file, this mode is not used.
'''
return 0o644
def add_key(self):
'''Add the GPG key(s) corresponding to this repo.
By default, if self.trustedparts_content contains content,
and self.trustedparts_file points to a file, the key(s) will
be added to the file.
If the file does not yet exist, and self.trustedparts_mode is set,
the file will be created with that mode.
'''
if self.deb822:
# The key is embedded into the file when the source is added.
return
if not all((self.trustedparts_file, self.trustedparts_content)):
return
dest = self.trustedparts_file
keys = self.trustedparts_content
if not isinstance(keys, bytes):
keys = keys.encode()
fp = self.fingerprints(keys)
print(_("Adding key to %s with fingerprint %s") % (dest, ','.join(fp)))
cmd = GPG_KEYRING_CMD % dest
action = "--import"
if not self.dry_run:
if not os.path.exists(dest):
# Create the dir if needed
if (dest.startswith(self.trustedparts_path)
and not os.path.exists(self.trustedparts_path)):
os.mkdir(self.trustedparts_path, mode=0o755)
if self.trustedparts_mode:
with open(dest, 'wb'):
os.chmod(dest, self.trustedparts_mode)
try:
with tempfile.TemporaryDirectory() as homedir:
cmd += f" --homedir {homedir} {action}"
subprocess.run(cmd.split(), check=True, input=keys)
except subprocess.CalledProcessError as e:
raise ShortcutException(e)
def remove_key(self):
'''Remove the GPG key(s) corresponding to this repo.
By default, if self.trustedparts_content contains content,
and self.trustedparts_file points to a file, the key(s) will
be removed from the file.
If the file contains no more keys after removal, the file will
be removed.
This does not consider other files; multiple repositories may
use the same signing key. This only modifies/removes
self.trustedparts_file.
'''
if not all((self.trustedparts_file, self.trustedparts_content)):
return
dest = self.trustedparts_file
fp = self.fingerprints(self.trustedparts_content)
if not os.path.exists(dest):
return
print(_("Removing key from %s with fingerprint %s") % (dest, ','.join(fp)))
cmd = GPG_KEYRING_CMD % dest
action = "--delete-keys %s" % ' '.join(fp)
if not self.dry_run:
try:
with tempfile.TemporaryDirectory() as homedir:
cmd += f" --homedir {homedir} {action}"
subprocess.run(cmd.split(), check=True)
except subprocess.CalledProcessError as e:
raise ShortcutException(e)
with open(dest, 'rb') as f:
empty = not self.fingerprints(f.read())
if empty:
os.remove(dest)
@property
def trustedparts_path(self):
'''Return result of apt_pkg.config.find_dir("Dir::Etc::trustedparts")'''
return apt_pkg.config.find_dir("Dir::Etc::trustedparts")
@property
def trustedparts_filename(self):
'''Get the trusted.gpg.d filename, without the leading path.'''
return self._filebase_to_filename('gpg')
@property
def trustedparts_file(self):
'''Get the trusted.gpg.d absolute-path filename.'''
return self._filename_to_file(self.trustedparts_path, self.trustedparts_filename)
@property
def trustedparts_content(self):
'''Content to put into trusted.gpg.d file'''
return None
@property
def trustedparts_mode(self):
'''Mode of trustedparts file'''
return 0o644
def add_login(self):
'''Add the login credentials corresponding to this repo.
By default, if self.netrcparts_content contains content,
and self.netrcparts_file points to a file, the file will be
created and content placed into it.
'''
if not all((self.netrcparts_file, self.netrcparts_content)):
return
dest = self.netrcparts_file
content = self.netrcparts_content
newfile = not os.path.exists(dest)
finalchar = '\n'
if not newfile:
with open(dest, 'r') as f:
lines = [l.strip() for l in f.readlines()]
with suppress(KeyError):
finalchar = lines[-1][-1]
if all([l.strip() in lines for l in content.splitlines()]):
print(_("Authentication data already in %s") % dest)
return
print(_("Adding authentication data to %s") % dest)
if not self.dry_run:
if newfile:
# Create the dir if needed
if (dest.startswith(self.netrcparts_path)
and not os.path.exists(self.netrcparts_path)):
os.mkdir(self.netrcparts_path, mode=0o755)
if self.netrcparts_mode:
with open(dest, 'w'):
os.chmod(dest, self.netrcparts_mode)
with open(dest, 'a') as f:
# we're appending; if the file doesn't end in \n, throw one in
if finalchar != '\n':
f.write('\n')
f.write(self.netrcparts_content)
def remove_login(self):
'''Remove the login credentials corresponding to this repo.
By default, if self.netrcparts_content contains content,
and self.netrcparts_file points to a file, the content will
be removed from the file.
If the file is empty (other than whitespace) after removal, the file
will be removed.
This does not consider other files; this only modifies/removes
self.netrcparts_file.
'''
if not all((self.netrcparts_file, self.netrcparts_content)):
return
dest = self.netrcparts_file
content = set([l.strip() for l in self.netrcparts_content.splitlines()])
if not os.path.exists(dest):
return
with open(dest, 'r') as f:
filecontent = set([l.strip() for l in f.readlines()])
if not filecontent & content:
print(_("Authentication data not contained in %s") % dest)
else:
print(_("Removing authentication data from %s") % dest)
if not self.dry_run:
with open(dest, 'w') as f:
f.write('\n'.join(filecontent - content))
if not self.dry_run:
with open(dest, 'r') as f:
empty = not f.read().strip()
if empty:
os.remove(dest)
@property
def netrcparts_path(self):
'''Return result of apt_pkg.config.find_dir("Dir::Etc::netrcparts")'''
return apt_pkg.config.find_dir("Dir::Etc::netrcparts")
@property
def netrcparts_filename(self):
'''Get the auth.conf.d filename, without the leading path.'''
return self._filebase_to_filename('conf')
@property
def netrcparts_file(self):
'''Get the auth.conf.d absolute-path filename.'''
return self._filename_to_file(self.netrcparts_path, self.netrcparts_filename)
@property
def netrcparts_content(self):
'''Content to put into auth.conf.d file
By default, if both username and password are set, this will return a proper
netrc-formatted line with the authentication information, including the
hostname and path.
'''
if not all((self.username, self.password)):
return None
hostname = urlparse(self.SourceEntry().uri).hostname
path = urlparse(self.SourceEntry().uri).path
return f'machine {hostname}{path} login {self.username} password {self.password}'
@property
def netrcparts_mode(self):
'''Mode of netrcparts file'''
return 0o600
def _set_source_entry(self, line):
'''Set the SourceEntry.
This should be called from subclasses to set the SourceEntry.
The SourceEntry file will be set to the sourceparts_file value.
The self.components, if any, will be added to the line's component(s).
'''
if self.deb822:
e = Deb822SourceEntry(line, file=self.sourceparts_file)
else:
e = SourceEntry(line, file=self.sourceparts_file)
e.comps = list(set(e.comps) | set(self.components))
self._source_entry = copy_source_entry(e)
def _encode_filebase(self, suffix=None):
base = self._filebase
if not base:
return None
if suffix:
base += '-%s' % suffix
return re.sub("[^a-z0-9_-]+", "_", base.lower())
def _filebase_to_filename(self, ext, suffix=None):
base = self._encode_filebase(suffix=suffix)
if not base:
return None
return '%s.%s' % (base, ext)
def _filename_to_file(self, path, name):
if not name:
return None
return os.path.join(path, name)
class ShortcutException(Exception):
'''General Exception during shortcut processing.'''
pass
class InvalidShortcutException(ShortcutException):
'''Invalid shortcut.
This should only be thrown from the constructor of a ShortcutHandler
subclass, and only to indicate that the provided shortcut is invalid
for that ShortcutHandler class.
'''
pass
# vi: ts=4 expandtab
Mini Shell