ansible-koji/roles/koji_hub/files/koji-sign-spooler

#!/usr/bin/python3
import os
import sys
import time
import glob
import logging
import configparser
import subprocess
import tempfile

# Setup logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(name)s: %(message)s',
    handlers=[
        logging.StreamHandler(sys.stdout)
    ]
)
logger = logging.getLogger('koji-sign-spooler')

SPOOL_DIR = '/var/spool/koji-sign/queue'
FAILED_DIR = '/var/spool/koji-sign/failed'

def get_siguldry_key_info():
    """
    Retrieves the public key and key ID/fingerprint from Siguldry.
    """
    env = os.environ.copy()
    env['SIGULDRY_SOCKET'] = '/run/siguldry-client-proxy/siguldry-client-proxy.socket'
    env['CREDENTIALS_DIRECTORY'] = '/etc/credstore'
    
    # Get key info
    res = subprocess.run(
        ['siguldry-client', '--credentials-directory', '/etc/credstore', '-c', '/etc/siguldry/client.toml', 'key', 'get', 'mxos-signing-key', '--certificates'],
        env=env,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True
    )
    if res.returncode != 0:
        logger.error(f"Failed to get key info from Siguldry: {res.stderr}")
        return None, None
        
    output = res.stdout
    start_marker = "-----BEGIN PGP PUBLIC KEY BLOCK-----"
    end_marker = "-----END PGP PUBLIC KEY BLOCK-----"
    
    start_idx = output.find(start_marker)
    end_idx = output.find(end_marker)
    if start_idx == -1 or end_idx == -1:
        logger.error("Could not find PGP public key block in Siguldry client output.")
        return None, None
        
    pubkey = output[start_idx:end_idx + len(end_marker)]
    
    with tempfile.NamedTemporaryFile(mode='w', suffix='.pub', delete=False) as f:
        f.write(pubkey)
        tmp_name = f.name
        
    try:
        gpg_res = subprocess.run(
            ['gpg', '--show-keys', '--with-colons', tmp_name],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            text=True
        )
        if gpg_res.returncode != 0:
            logger.error(f"Failed to read GPG key ID: {gpg_res.stderr}")
            return None, None
            
        key_id = None
        fingerprint = None
        for line in gpg_res.stdout.splitlines():
            parts = line.split(':')
            if parts[0] == 'pub':
                key_id = parts[4][-8:]
            elif parts[0] == 'fpr':
                fingerprint = parts[9]
                break
                
        sig_key_id = fingerprint if fingerprint else key_id
        return pubkey, sig_key_id
    finally:
        try:
            os.remove(tmp_name)
        except Exception:
            pass

def process_queue():
    if not os.path.exists(SPOOL_DIR):
        return

    # Import kojihub dynamically so that it loads after service starts
    import kojihub
    import koji

    # Read configuration file to get KojiDir and DB options
    config = configparser.ConfigParser()
    config_files = ['/etc/koji-hub/hub.conf'] + sorted(glob.glob('/etc/koji-hub/hub.conf.d/*.conf'))
    config.read(config_files)
    
    koji_dir = config.get('hub', 'KojiDir', fallback='/mnt/koji')

    # Connect to database using configuration parameters
    db_opts = {}
    if config.has_option('hub', 'DBConnectionString'):
        db_opts['dsn'] = config.get('hub', 'DBConnectionString')
    else:
        if config.has_option('hub', 'DBName'):
            db_opts['database'] = config.get('hub', 'DBName')
        if config.has_option('hub', 'DBUser'):
            db_opts['user'] = config.get('hub', 'DBUser')
        if config.has_option('hub', 'DBPass'):
            db_opts['password'] = config.get('hub', 'DBPass')
        if config.has_option('hub', 'DBHost'):
            db_opts['host'] = config.get('hub', 'DBHost')
        elif config.has_option('hub', 'DBhost'):
            db_opts['host'] = config.get('hub', 'DBhost')
        if config.has_option('hub', 'DBPort'):
            try:
                db_opts['port'] = config.getint('hub', 'DBPort')
            except ValueError:
                pass
    
    db_opts = {k: v for k, v in db_opts.items() if v is not None}
    kojihub.db.setDBopts(**db_opts)

    try:
        kojihub.context.cnx = kojihub.db.connect()
    except Exception as e:
        logger.error(f"Failed to connect to Koji Hub DB: {e}")
        return

    # Get list of files in spool directory
    files = sorted(os.listdir(SPOOL_DIR))
    if not files:
        return

    for filename in files:
        file_path = os.path.join(SPOOL_DIR, filename)
        if not os.path.isfile(file_path):
            continue

        try:
            build_id = int(filename)
        except ValueError:
            logger.error(f"Invalid build ID in spool file name: {filename}")
            os.rename(file_path, os.path.join(FAILED_DIR, filename))
            continue

        with open(file_path, 'r') as f:
            build_nvr = f.read().strip()

        logger.info(f"Processing build {build_id} ({build_nvr}) for signing")

        # Retry loop to wait for the completeBuild transaction to commit
        rpms = []
        for attempt in range(5):
            try:
                rpms = kojihub.list_rpms(buildID=build_id)
                if rpms:
                    break
            except Exception as e:
                logger.warning(f"Error querying RPMs (attempt {attempt+1}): {e}")
            logger.info(f"RPMs not found yet for build {build_id}, waiting 1 second... (attempt {attempt+1}/5)")
            time.sleep(1)

        if not rpms:
            logger.error(f"No RPMs found for build {build_id} after retries. Moving spool file to failed.")
            os.makedirs(FAILED_DIR, exist_ok=True)
            os.rename(file_path, os.path.join(FAILED_DIR, filename))
            continue

        # Get Siguldry key information
        pubkey, sig_key_id = get_siguldry_key_info()
        if not sig_key_id:
            logger.error("Failed to retrieve Siguldry key ID. Skipping build signing.")
            os.makedirs(FAILED_DIR, exist_ok=True)
            os.rename(file_path, os.path.join(FAILED_DIR, filename))
            continue

        # Write public key to koji topdir so it can be served
        pubkey_path = os.path.join(koji_dir, 'gpg_key.pub')
        try:
            with open(pubkey_path, 'w') as f:
                f.write(pubkey)
            logger.info(f"Wrote public GPG key to {pubkey_path}")
        except Exception as e:
            logger.warning(f"Failed to write public key to {pubkey_path}: {e}")

        # Get build info and construct RPM path
        try:
            build_info = kojihub.get_build(build_id)
            build_dir = koji.pathinfo.build(build_info)
        except Exception as e:
            logger.error(f"Failed to get build info for {build_id}: {e}")
            os.makedirs(FAILED_DIR, exist_ok=True)
            os.rename(file_path, os.path.join(FAILED_DIR, filename))
            continue

        success = True
        for rpm in rpms:
            rpm_id = rpm['id']
            rpm_nvr = rpm['nvr']
            # Construct RPM path using koji.pathinfo
            rpm_rel_path = koji.pathinfo.rpm(rpm)
            full_rpm_path = os.path.join(koji_dir, build_dir, rpm_rel_path)

            try:
                logger.info(f"Signing RPM {rpm_nvr} via rpmsign and Siguldry PKCS#11")
                env = os.environ.copy()
                env['PKCS11_MODULE_PATH'] = '/usr/lib64/pkcs11/libsiguldry_pkcs11.so'
                env['SIGULDRY_SOCKET'] = '/run/siguldry-client-proxy/siguldry-client-proxy.socket'
                env['CREDENTIALS_DIRECTORY'] = '/etc/credstore'

                # Sign using rpmsign command with GnuPG backend using PKCS#11 module
                cmd = [
                    'rpmsign',
                    '--define', f'_gpg_name {sig_key_id}',
                    '--addsign', full_rpm_path
                ]
                
                result = subprocess.run(
                    cmd,
                    env=env,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    text=True
                )

                if result.returncode != 0:
                    logger.error(f"rpmsign failed for {rpm_nvr}: stdout: {result.stdout}, stderr: {result.stderr}")
                    success = False
                else:
                    logger.info(f"Successfully signed RPM {rpm_nvr} on disk. Extracting signature header.")
                    sig_data = koji.rip_rpm_sighdr(full_rpm_path)
                    kojihub.add_rpm_sig(rpm_id, sig_data)
                    logger.info(f"Successfully imported signature for {rpm_nvr} with key {sig_key_id}")
            except Exception as e:
                logger.error(f"Error signing RPM {rpm_nvr}: {e}")
                success = False

        if success:
            try:
                kojihub.context.cnx.commit()
                logger.info(f"All signatures committed for build {build_id} ({build_nvr}).")

                # Check tags and trigger dist-repo if tagged in mxos0
                try:
                    tags = list(kojihub.list_tags(build=build_id))
                    tag_names = [t['name'] for t in tags]
                    logger.info(f"Build {build_nvr} is tagged in: {tag_names}")
                    if 'mxos0' in tag_names:
                        short_key_id = sig_key_id[-8:].lower()
                        logger.info(f"Triggering dist-repo for tag mxos0 with key {short_key_id}")
                        dist_cmd = [
                            '/usr/local/bin/mxos-trigger-dist-repo.sh', short_key_id
                        ]
                        subprocess.Popen(dist_cmd)
                except Exception as e:
                    logger.error(f"Failed to check tags/trigger dist-repo: {e}")

                os.remove(file_path)
            except Exception as e:
                logger.error(f"Failed to commit signatures to database: {e}")
                try:
                    kojihub.context.cnx.rollback()
                except Exception:
                    pass
        else:
            logger.error(f"Failed to sign some/all RPMs for build {build_id}. Keeping spool file for retry.")
            break

    try:
        kojihub.context.cnx.close()
    except Exception:
        pass

def main():
    logger.info("Koji Sign Spooler Daemon started")
    os.makedirs(SPOOL_DIR, exist_ok=True)
    os.makedirs(FAILED_DIR, exist_ok=True)
    
    while True:
        try:
            process_queue()
        except Exception as e:
            logger.error(f"Unhandled exception in spooler loop: {e}")
        time.sleep(5)

if __name__ == '__main__':
    main()