#!/usr/bin/perl -w
#
# Copyright (c) 2006-2009 Michael Schroeder, Novell Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program (see the file COPYING); if not, write to the
# Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#
################################################################
#
# Worker build process. Builds jobs received from a Repository Server,
# sends build binary packages back.
#

BEGIN {
  my ($wd) = $0 =~ m-(.*)/- ;
  $wd ||= '.';
  unshift @INC,  "$wd/build";
  unshift @INC,  "$wd";
  $XML::Structured::pureperl = 1 unless defined $XML::Structured::pureperl;
}

use Digest::MD5 ();
use Digest::SHA ();
use XML::Structured ':bytes';
use Data::Dumper;
use POSIX;
use Storable;
use Fcntl qw(:DEFAULT :flock);
BEGIN { Fcntl->import(':seek') unless defined &SEEK_SET; }

use BSConfiguration;
use BSOBS;
use BSRPC;
use BSServer;
use BSDispatch;
use BSUtil;
use BSXML;
use BSHTTP;
use BSBuild;
use BSCando;
use BSVerify;

my $can_rewrite_containers;
eval {
  require BSTar;
  require BSContar;
  $can_rewrite_containers = 1;
};

eval { BSRPC::import(':https') };

use strict;

my @binsufs = @BSOBS::binsufs;
my $binsufsre = join('|', map {"\Q$_\E"} @binsufs);

my $buildroot;
my $port;
my $proto;
my $bindaddress;
my $statedir;
my $hostarch;
my $vm = '';
my $vm_tmpfs_mode;
my $vm_root = '';
my $vm_swap = '';
my $vm_kernel;
my $vm_initrd;
my $vm_memory;
my $vm_custom_option;
my $vm_use_mkfs_copyin;
my $vm_enable_console;
my $vm_worker_name;
my $vm_worker_instance;
my $vmdisk_rootsize;
my $vmdisk_swapsize;
my $vmdisk_filesystem;
my $vmdisk_filesystem_options;
my $vmdisk_mount_options;
my $vmdisk_clean;
my $vm_network;
my $emulator_script;
my $hugetlbfs;
my $workerid;
my $srcserver;
my @reposervers;
my $testmode;
my $testcasemode;
my $noworkercheck;
my $nobuildcodecheck;
my $oneshot;
my $silent;
my $hostcheck;
my $localkiwi;
my $owner;
my @hostlabel;
my $getbinariesproxy;
my $getbinariesproxy_kiwiproduct;
my $hardstatus;

my $jobs;
my $threads;
my $cachedir;
my $cachesize;
my $cleanup_chroot;
my $wipeafterbuild;
my $openstack_flavor;
my $vm_server;

# current XEN has no xenstore anymore
my $xenstore_maxsize = 20 * 1000000;
# 1 hour timeout to avoid forever hanging workers
my $gettimeout = 1 * 3600;

my $buildlog_maxsize = $BSConfig::buildlog_maxsize ? $BSConfig::buildlog_maxsize : 500 * 1000000;
my $buildlog_maxidle = $BSConfig::buildlog_maxidle ? $BSConfig::buildlog_maxidle :   8 * 3600;

# for collecting statistics
my $binariesdownload;
my $binariescachehits;
my $binariesdownloadsize;


my $workercode;
my $buildcode;

$hostcheck = $BSConfig::workerhostcheck if defined($BSConfig::workerhostcheck);

sub lockstate {
  while (1) {
    open(STATELOCK, '>>', "$statedir/state") || die("$statedir/state: $!\n");
    flock(STATELOCK, LOCK_EX) || die("flock $statedir/state: $!\n");
    my @s = stat(STATELOCK);
    last if $s[3];	# check nlink
    close(STATELOCK);	# race, try again
  }
  my $oldstate = readxml("$statedir/state", $BSXML::workerstate, 1);
  $oldstate = {} unless $oldstate;
  return $oldstate;
}

sub unlockstate {
  close(STATELOCK);
}

sub commitstate {
  my ($newstate) = @_;
  writexml("$statedir/state.new", "$statedir/state", $newstate, $BSXML::workerstate) if $newstate;
  close(STATELOCK);
}

sub hardstatus {
  return unless $hardstatus;
  my $l = $_[0] || '';
  $l =~ s/[\000-\037\177-\377]//g;	# ascii only, please
  $l = substr($l, 0, 40) if length($l) > 40;
  print "\033_$l\033\\";
}

sub trunc_logfile {
  my ($lf) = @_;
  open(LF, '<', $lf) || return; 
  my $buf;
  sysread(LF, $buf, 1000000);
  $buf .= "\n\n[truncated]\n\n";
  sysseek(LF, -1000000, 2);
  sysread(LF, $buf, 1000000, length($buf));
  close LF;
  $buf .= "\nLogfile got too big, killed job.\n";
  open(LF, ">$lf.new") || return; 
  syswrite(LF, $buf);
  close LF;
  rename("$lf.new", $lf);
}

sub tail_logfile {
  my ($lf, $size) = @_;
  local *LF;
  open(LF, '<', $lf) || return ''; 
  if (-s LF > $size) {
    defined(sysseek(LF, -$size, 2)) || return '';
  }
  my $buf = '';
  sysread(LF, $buf, $size);
  close LF;
  return $buf;
}

sub cleanup_job {
  if ($vm_tmpfs_mode) {
    # no need to umount if $buildroot is already umounted
    if (not qsystem('mountpoint', '-q', $buildroot)) {
      qsystem("umount", "-l", $buildroot) && die("umount tmpfs failed: $!\n");
    }
  }

  # if this is a chroot like case, get rid of it after the build
  if ((!$vm || $vm eq 'lxc' || $vm eq 'docker' || $vm eq 'nspwan') && $cleanup_chroot && -d $buildroot) {
      print "Removing buildroot $buildroot...";
      eval {
        rm_rf($buildroot);
        mkdir_p($buildroot);
      };
      print "\n";
  }

}

sub kill_job {
  my @args;
  # same args as in the build call
  if ($vm =~ /(xen|kvm|qemu|zvm|emulator|pvm|openstack)/) {
    push @args, '--root', "$buildroot/.mount";
    push @args, '--vm-type', $vm;
    push @args, '--vm-disk', $vm_root;
    push @args, '--emulator-script', $emulator_script if $vm eq 'emulator' && $emulator_script;
    push @args, '--vm-worker', $vm_worker_name if $vm_worker_name;
    push @args, '--vm-worker-nr', $vm_worker_instance if $vm_worker_instance;
  } else {
    push @args, '--root', $buildroot;
    push @args, '--vm-type', $vm if $vm eq 'lxc' || $vm eq 'docker';
  }
  if (system("$statedir/build/build", @args, "--kill")) {
    return 0;
  }
  cleanup_job();
  return 1;
}

sub sysrq_job {
  my ($sysrq) = @_;
  my @args;
  if ($vm =~ /(xen|kvm|qemu|zvm|emulator|pvm|openstack)/) {
    push @args, '--root', "$buildroot/.mount";
    push @args, '--vm-type', $vm;
    push @args, '--vm-disk', $vm_root;
    push @args, '--vm-swap', $vm_swap;
  } else {
    # can not work for for chroot and container builds
    return 1;
  }
  if (system("$statedir/build/build", @args, "--sysrq", $sysrq)) {
    return 0;
  }
  return 1;
}

sub wipe_all {
  my @args;
  if ($vm =~ /(xen|kvm|qemu|zvm|emulator|pvm|openstack)/) {
    push @args, '--root', "$buildroot/.mount";
    push @args, '--vm-type', $vm;
    push @args, '--vm-disk', $vm_root;
    push @args, '--vm-swap', $vm_swap;
  } else {
    push @args, '--root', $buildroot;
  }
  if (system("$statedir/build/build", @args, "--wipe")) {
    return 0;
  }
  return 1;
}

my @saveargv = @ARGV;	# so we can restart ourself

sub usage {
  my ($ret) = @_;

print <<EOF;
Usage: $0 [OPTION] --root <directory> --statedir <directory>

       --root      : buildroot directory

       --port      : fixed port number

       --statedir  : state directory

       --id        : worker id

       --reposerver: define reposerver, can be used multiple times

       --arch      : define hostarch (overrides 'uname -m')
                     currently supported architectures: 
                     @{[sort keys %BSCando::cando]}

       --emulator  : enable emulator (requires prepared emulator.sh script)
       
       --zvm       : enable z/VM

       --tmpfs     : uses tmpfs (memory) for the build root

       --device    : set kvm or xen root device (default is <root>/root file)

       --swap      : set kvm or xen swap device (default is <root>/swap file)

       --hostlabel : define a host label for build constraints, can be used multiple times

       --vm-type   : set virtualisation mode to use (kvm, xen, qemu:\$HOSTARCH, zvm, emulator:\$HOSTARCH)
                     Also possible are lxc or docker, but these are less secure and are not using
                     the specified kernel for the build target.

       --vm-kernel : set kernel to use (xen/kvm)

       --vm-initrd : set initrd to use (xen/kvm)

       --vm-memory : set amount of memory to use (xen/kvm/lxc)

       --vm-worker NAME
                   : (z/VM) set name of the actual worker

       --vm-worker-nr NUMBER
                   : (z/VM) set instance number of the actual worker

       --vm-custom-option SWITCHES
                   : hand over custom switches to vm handler (eg. qemu)

       --vmdisk-rootsize <size>
                   : size of the root disk image (default 4096M)

       --vmdisk-swapsize <size>
                   : size of the swap disk image (default 1024M)

       --vmdisk-filesystem <none|ext3|ext4>
                   : filesystem to use for autosetup root disk image

       --vm-network: enable network (kvm)

       --emulator-script <script>
                   : Define the emulator script 

       --hugetlbfs HUGETLBFSPATH
                   : Use hugetlb for memory management, path to mounted hugetlbfs.

       --test      : enable test mode

       --build     : just build the package, don't send anything back
                     (needs a buildinfo file as argument)

       --noworkerupdate
                   : do not check if the worker is up-to-date

       --nobuildupdate
                   : do not check if the build code is up-to-date

       --nocodeupdate
                   : do not update both worker and build code

       --jobs <nr> : hand over the number of parallel jobs to build

       --threads <nr> : sets number of threads for KVM

       --oneshot <seconds>
                   : just build one package, do not wait more then
                     <seconds> seconds if nothing is available

       --hostcheck <hostcheck>
                   : call to check if the host can build the package

       --owner <name>
                   : report the build host owner name to server

       --cachedir <cachedir>
       --cachesize <size_in_mb>
                   : use cachedir to cache fetched binaries

       --getbinariesproxy <proxyserver>
       --getbinariesproxy-kiwiproduct <proxyserver>
                   : define a proxy server for getbinaries requests

       --help      : this message

Your call:

$0 @saveargv

EOF
  exit($ret || 0);
}

my $justbuild;
my $exitrestart;

my $exitrestart_timeout = 300;	# wait max 5 minuntes

exit(0) if @ARGV == 1 && $ARGV[0] eq '--selftest';
if ($ARGV[0] eq '--testcase') {
  @ARGV= ();
  $testcasemode = 1;
  $buildroot = '/testcasemode/buildroot';
  $statedir = '/testcasemode/statedir';
  $hostarch = 'x86_64';
}

while (@ARGV) {
  usage(0) if $ARGV[0] eq '--help';
  if ($ARGV[0] eq '--exit' || $ARGV[0] eq '--stop') {
    shift @ARGV;
    $exitrestart = 'exit';
    next;
  }
  if ($ARGV[0] eq '--restart') {
    shift @ARGV;
    $exitrestart = 'restart';
    next;
  }
  if ($ARGV[0] eq '--root') {
    shift @ARGV;
    $buildroot = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--port') {
    shift @ARGV;
    $port = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--bindaddress') {
    shift @ARGV;
    $bindaddress = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--proto') {
    shift @ARGV;
    $proto = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--arch') {
    shift @ARGV;
    $hostarch = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--statedir') {
    shift @ARGV;
    $statedir = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--srcserver') {
    # default value used if buildinfo does not contain srcserver element
    shift @ARGV;
    $srcserver = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--reposerver') {
    shift @ARGV;
    my $server = shift @ARGV;
    push @reposervers, $server unless grep {$_ eq $server} @reposervers;
    next;
  }
  if ($ARGV[0] eq '--getbinariesproxy') {
    shift @ARGV;
    $getbinariesproxy = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--getbinariesproxy-kiwiproduct') {
    shift @ARGV;
    $getbinariesproxy_kiwiproduct = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--hostlabel') {
    shift @ARGV;
    my $label = shift @ARGV;
    push @hostlabel, $label unless grep {$_ eq $label} @hostlabel;
    next;
  }
  if ($ARGV[0] eq '--id') {
    shift @ARGV;
    $workerid = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--test') {
    shift @ARGV;
    $testmode = 1;
    next;
  }
  if ($ARGV[0] =~ /^--(kvm|xen|lxc|emulator|zvm|docker|pvm|openstack)$/) {
    $vm = $1;
    shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--tmpfs') {
    shift @ARGV;
    $vm_tmpfs_mode = 1;
    next;
  }
  if ($ARGV[0] eq '--xendevice' || $ARGV[0] eq '--device') {
    shift @ARGV;
    $vm_root = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--xenswap' || $ARGV[0] eq '--swap') {
    shift @ARGV;
    $vm_swap = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-kernel') {
    shift @ARGV;
    $vm_kernel = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-initrd') {
    shift @ARGV;
    $vm_initrd = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-memory') {
    shift @ARGV;
    $vm_memory = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-custom-option') {
    shift @ARGV;
    $vm_custom_option = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-worker') {
    shift @ARGV;
    $vm_worker_name = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-worker-nr') {
    shift @ARGV;
    $vm_worker_instance = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-network') {
    shift @ARGV;
    $vm_network = 1;
    next;
  }
  if ($ARGV[0] eq '--vm-enable-console') {
    shift @ARGV;
    $vm_enable_console = 1;
    next;
  }
  if ($ARGV[0] eq '--vm-use-mkfs-copyin') {
    shift @ARGV;
    $vm_use_mkfs_copyin = 1;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-rootsize') {
    shift @ARGV;
    $vmdisk_rootsize = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-swapsize') {
    shift @ARGV;
    $vmdisk_swapsize = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-filesystem') {
    shift @ARGV;
    $vmdisk_filesystem = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-filesystem-options') {
    shift @ARGV;
    $vmdisk_filesystem_options = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-mount-options') {
    shift @ARGV;
    $vmdisk_mount_options = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vmdisk-clean') {
    shift @ARGV;
    $vmdisk_clean = 1;
    next;
  }
  if ($ARGV[0] eq '--emulator-script') {
    shift @ARGV;
    $emulator_script = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--hugetlbfs') {
    shift @ARGV;
    $hugetlbfs = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--build') {
    shift @ARGV;
    $justbuild = 1;
    next;
  }
  if ($ARGV[0] eq '--oneshot') {
    shift @ARGV;
    $oneshot = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--hostcheck') {
    shift @ARGV;
    $hostcheck = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--nocodeupdate') {
    shift @ARGV;
    $noworkercheck = 1;
    $nobuildcodecheck = 1;
    next;
  }
  if ($ARGV[0] eq '--noworkerupdate') {
    shift @ARGV;
    $noworkercheck = 1;
    next;
  }
  if ($ARGV[0] eq '--nobuildupdate') {
    shift @ARGV;
    $nobuildcodecheck = 1;
    next;
  }
  if ($ARGV[0] eq '--silent') {
    shift @ARGV;
    $silent= 1;
    next;
  }
  if ($ARGV[0] eq '--localkiwi') {
    shift @ARGV;
    $localkiwi = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--owner') {
    shift @ARGV;
    $owner = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--jobs') {
    shift @ARGV;
    $jobs = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--threads') {
    shift @ARGV;
    $threads = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--cachedir') {
    shift @ARGV;
    $cachedir = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--cachesize') {
    shift @ARGV;
    $cachesize = shift @ARGV;
    $cachesize *= 1024*1024;
    next;
  }
  if ($ARGV[0] eq '--hardstatus') {
    shift @ARGV;
    $hardstatus = 1;
    next;
  }
  if ($ARGV[0] eq '--cleanup-chroot') {
    shift @ARGV;
    $cleanup_chroot = 1;
    next;
  }
  if ($ARGV[0] eq '--wipeafterbuild') {
    shift @ARGV;
    $wipeafterbuild = 1;
    next;
  }
  if ($ARGV[0] eq '--openstack-flavor') {
    shift @ARGV;
    $openstack_flavor = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-server') {
    shift @ARGV;
    $vm_server = shift @ARGV;
    next;
  }
  if ($ARGV[0] eq '--vm-type') {
    shift @ARGV;
    $vm = shift @ARGV;
    next;
  }
  print "OPTION UNKNOWN: $ARGV[0]\n" unless $justbuild;
  last;
}

if ($exitrestart) {
  usage(1) unless $statedir;
  if ((! -e "$statedir/lock") || BSUtil::lockcheck('>>', "$statedir/lock")) {
    die("worker not running.\n") if $exitrestart eq 'restart';
    print "worker not running.\n";
    exit(0);
  }
  my $state = lockstate();
  $state->{'state'} = 'idle' unless $state->{'state'};
  if ($state->{'state'} eq 'building' || $state->{'state'} eq 'killed' || $state->{'state'} eq 'discarded' || $state->{'state'} eq 'badhost') {
    $state->{'state'} = 'discarded';
    $state->{'nextstate'} = $exitrestart eq 'exit' ? 'exit' : 'rebooting';
  } elsif ($state->{'state'} eq 'dead') {
    die("worker is dead\n");
  } else {
    $state->{'state'} = $exitrestart eq 'exit' ? 'exit' : 'rebooting';
  }
  commitstate($state);
  # now wait till the state changes...
  while(1) {
    select(undef, undef, undef, .1);
    if (defined($exitrestart_timeout)) {
      $exitrestart_timeout -= .1;
      die("timeout reached, worker not responding\n") if $exitrestart_timeout < 0;
    }
    my $curstate = readxml("$statedir/state", $BSXML::workerstate, 1);
    last unless $curstate && $curstate->{'state'};
    next if $curstate->{'nextstate'};
    last if $curstate->{'state'} ne 'exit' && $curstate->{'state'} ne 'rebooting';
  }
  exit(0);
}

usage(1) unless $buildroot && $statedir;
usage(1) if ($cachedir && !$cachesize) || ($cachesize && !$cachedir);

# default needs to be in sync with build(1)
# otherwise job killing does not work in a default setup
$vm_root = "${buildroot}.img" unless $vm_root;
$vm_swap = "${buildroot}.swap" unless $vm_swap;

# here's the build code we want to use
$::ENV{'BUILD_DIR'} = "$statedir/build";

if (!$hostarch) {
  $hostarch = `uname -m`;
  chomp $hostarch;
  die("could not determine hostarch\n") unless $hostarch;
}

die("arch $hostarch cannot build anything!\n") unless $BSCando::cando{$hostarch} || ($hostarch eq 'local' && $localkiwi);

$srcserver = $BSConfig::srcserver unless defined $srcserver;
@reposervers = @BSConfig::reposervers unless @reposervers;
@reposervers = $BSConfig::reposerver unless @reposervers;

if ($justbuild) {
  my $buildinfo = readxml($ARGV[0], $BSXML::buildinfo);
  if ($localkiwi) {
    # make sure this is the right job for us
    my $imagemode = getimagemode($buildinfo);
    die("not a product job\n") unless $imagemode && ($imagemode eq 'kiwiproduct' || $imagemode eq 'productcompose');
  }
  $| = 1;
  dobuild($buildinfo);
  exit(0);
}

# collect environment information for the dispatcher. To be checked with job constraints
sub device_size($) {
  my ($device) = @_;
  if ($device) {
    if (-b $device) {
      if (open(SIZE, '-|', '/sbin/blockdev', '--getsize64', $device)) {
        my $line = <SIZE>;
        chomp $line;
        my $size = $line / ( 1024 * 1024 );
        close SIZE;
        return $size;
      }
    } else {
      # implement me for loop device files
    }
  }
  return undef;
}

# create worker info data
my $workerinfo = {};
# host labels
$workerinfo->{'hostlabel'} = \@hostlabel if @hostlabel;
# sandbox
$workerinfo->{'sandbox'} = $vm || 'chroot';
# build host owner
$workerinfo->{'owner'} = $owner if $owner;
# linux kernel
my $str = readstr('/proc/version');
if ($str =~ /^Linux version ([^ ]*)-([^- ]*) /s) {
  $workerinfo->{'linux'} = { 'version' => "$1", 'flavor' => $2 };
}
my $hw = {'cpu' => {}, 'processors' => 0};
# memory, undefined in non vm mode since no guarantees are exist
if (open(FILE, "<", "/proc/cpuinfo")) {
  my $cpu_implementer;
  my $cpu_variant;
  my $cpu_part;

  while(<FILE>) {
    chomp;
    if (/^processor/s) {
      $hw->{'processors'} = $hw->{'processors'} + 1;
    } elsif (/^flags\s*:\s(.*)$/) {
      # classic ix86 and x86_64
      my @cpuflags = split(' ', $1);
      $hw->{'cpu'}->{'flag'} = \@cpuflags;
    } elsif (/^features\s*:\s(.*)$/) {
      # s390 Z-System
      my @cpuflags = split(' ', $1);
      $hw->{'cpu'}->{'flag'} = \@cpuflags;
    } elsif (/^Features\s*:\s(.*)$/) {
      # aarch64
      my @cpuflags = split(' ', $1);
      $hw->{'cpu'}->{'flag'} = \@cpuflags;
    } elsif (/^CPU implementer\s*:\s(.*)$/) {
      $cpu_implementer = $1;
    } elsif (/^CPU variant\s*:\s(.*)$/) {
      $cpu_variant = $1;
    } elsif (/^CPU part\s*:\s(.*)$/) {
      $cpu_part = $1;
    } elsif (/^cpu\s*:\sPOWER9/) {
      # PowerPC kernel provides no flags, but we need to be able
      # to distinguish between power7 and 8 at least
      $hw->{'cpu'}->{'flag'} = [ "power7", "power8", "power9" ];
    } elsif (/^cpu\s*:\sPOWER8/) {
      # PowerPC kernel provides no flags, but we need to be able
      # to distinguish between power7 and 8 at least
      $hw->{'cpu'}->{'flag'} = [ "power7", "power8" ];
    } elsif (/^cpu\s*:\sPOWER7/) {
      $hw->{'cpu'}->{'flag'} = [ "power7" ];
    }
  }
  close FILE;

  # Check for aarch64/aarch32 compatibility
  if ($hostarch eq 'aarch64' && $cpu_implementer) {
    my $supports_aarch32;
    my $require_aarch64_kernel;

    if ($cpu_implementer eq "0x50") {
      # Applied Micro / Ampere Computing
      if ($cpu_variant eq "0x0") {
        # APM X-Gene 1 supports aarch32
        $supports_aarch32 = 1;
      } elsif ($cpu_variant eq "0x1") {
        # APM X-Gene 2 supports aarch32
        $supports_aarch32 = 1;
      } elsif ($cpu_variant eq "0x3") {
        # Ampere eMAG supports aarch32
        $supports_aarch32 = 1;
      }
    } elsif ($cpu_implementer eq "0x41") {
      # ARM Inc.
      if ($cpu_part eq '0xd0c' || $cpu_part eq '0xd0e') {
        # Neoverse N1 || Cortex A76AE
        # 32bit user land is supported, but only on a 64bit kernel
        $require_aarch64_kernel = 1;
        $supports_aarch32 = 1;
      } elsif ($cpu_part lt '0xd0c') {
        # old hardware had full support of 32bit
        $supports_aarch32 = 1;
      }
      # otherwise we assume 64bit only for all future hardware
    }

    $hw->{'nativeonly'} = undef unless $supports_aarch32;
    push @{$hw->{'cpu'}->{'flag'}}, 'EL0' if $require_aarch64_kernel;
  }

  # Extend flags with architecture level for x86_64
  if ($hostarch eq 'x86_64') {
    my %flags = map {$_ => 1} @{$hw->{'cpu'}->{'flag'}};
    push @{$hw->{'cpu'}->{'flag'}}, 'x86-64-v2' unless grep {!$flags{$_}} qw{cx16 lahf_lm popcnt pni sse4_1 sse4_2 ssse3};
    push @{$hw->{'cpu'}->{'flag'}}, 'x86-64-v3' unless grep {!$flags{$_}} qw{avx2 bmi1 bmi2 f16c fma}; # linux is not offering flags for LZCNT, MOVBE and OSXSAVE
    push @{$hw->{'cpu'}->{'flag'}}, 'x86-64-v4' unless grep {!$flags{$_}} qw{avx512f avx512dq avx512cd avx512bw avx512vl};
  }
}
$hw->{'jobs'} = $jobs if $jobs;
$hw->{'memory'} = $vm_memory if $vm_memory;
$hw->{'swap'} = $vmdisk_swapsize if $vmdisk_swapsize;
$hw->{'disk'} = $vmdisk_rootsize if $vmdisk_rootsize;
if ($vm_root) {
  my $size = device_size($vm_root);
  $hw->{'disk'} = $size if $size;
}
if ($vm_swap) {
  my $size = device_size($vm_swap);
  $hw->{'swap'} = $size if $size;
}

if ($vm =~ /openstack/) {
  $hw->{'processors'} = $hw->{'jobs'} = $jobs if $jobs;
  $hw->{'memory'} = $vm_memory if $vm_memory;
  $hw->{'swap'} = $vmdisk_swapsize if $vmdisk_swapsize;
  $hw->{'disk'} = $vmdisk_rootsize if $vmdisk_rootsize;
}

$workerinfo->{'hardware'} = $hw;

sub stream_logfile {
  my ($jobid, $nostream, $start, $end) = @_;
  $start ||= 0;
  if (defined($end)) {
    die("start cannot be negative if an end position is specified\n") if $start < 0;
    $end -= $start;
    die("end is smaller than start\n") if $end < 0;
  }

  open(F, '<', "$buildroot/.build.log") || die("$buildroot/.build.log: $!\n");
  my @s = stat(F);
  die("Logfile is not that big\n") if $s[7] < abs($start);
  my $pos = sysseek(F, $start, $start < 0 ? Fcntl::SEEK_END : Fcntl::SEEK_SET);
  die("sysseek: $!\n") unless defined $pos;

  BSServer::reply(undef, 'Content-Type: text/plain', 'Transfer-Encoding: chunked');
  while (!defined($end) || $end) {
    @s = stat(F);
    last unless @s;
    if ($s[7] <= $pos) {
      if (!$s[3]) {
	# logfile was deleted or replaced
	if ($pos == 0) {
          select(undef, undef, undef, .5);
	  if (-e "$buildroot/.build.log") {
	    close(F);
	    open(F, '<', "$buildroot/.build.log") || die("$buildroot/.build.log: $!\n");
	    eval { $jobid && getstate($jobid, 1) };
	    next unless $@;
	  }
	}
        last;
      }
      select(undef, undef, undef, .5);
      next;
    }
    my $l = $s[7] - $pos;
    $l = 4096 if $l > 4096;
    my $data = '';
    sysread(F, $data, $l);
    next unless length($data);
    $data = substr($data, 0, $end) if defined($end) && length($data) > $end;
    $pos += length($data);
    $end -= length($data) if defined $end;
    $data = sprintf("%X\r\n", length($data)).$data."\r\n";
    BSServer::swrite($data);
    last if $nostream && $pos >= $s[7];
  }
  close F;
  BSServer::swrite("0\r\n\r\n");
}

sub send_state {
  my ($state, $p, $ba, $exclude) = @_;
  my @args = ("state=$state", "arch=$ba", "port=$p");
  push @args, "proto=$proto" if ($proto || 'http') ne 'http';
  push @args, "workerid=$workerid" if defined $workerid;
  for my $server (@reposervers) {
    next if $exclude && $server eq $exclude;
    if ($state eq 'idle' && @reposervers > 1) {
      my $curstate = readxml("$statedir/state", $BSXML::workerstate, 1);
      last if $curstate && $curstate->{'state'} ne $state;
    }
    my $param = {
      'uri' => "$server/worker",
      'timeout' => 3,
    };
    if ($state eq 'idle') {
      $param->{'request'} = 'POST';
      $param->{'data'} = BSUtil::toxml({ %$workerinfo, 'registerserver' => $server}, $BSXML::worker);
    }
    eval {
      BSRPC::rpc($param, undef, @args);
    };
    print "send_state $server: $@" if $@;
  }
}

sub send_dispatched {
  my ($server, $p, $ba, $buildinfo) = @_;
  my @args = ("hostarch=$ba", "port=$p");
  push @args, "proto=$proto" if ($proto || 'http') ne 'http';
  push @args, "workerid=$workerid" if defined $workerid;
  push @args, "job=$buildinfo->{'job'}";
  push @args, "arch=$buildinfo->{'arch'}";
  push @args, "jobid=$buildinfo->{'jobid'}";
  my $param = {
    'uri' => "$server/workerdispatched",
    'request' => 'POST',
    'timeout' => 600,
  };
  BSRPC::rpc($param, undef, @args);
}

sub codemd5 {
  my ($dir) = @_;
  my @files = grep {!/^\./} ls($dir);
  my @bfiles = map {"Build/$_"} grep {!/^\./} ls("$dir/Build");
  my @efiles = map {"emulator/$_"} grep {!/^\./} ls("$dir/emulator");
  @files = sort(@files, @bfiles, @efiles);
  my $md5 = '';
  for my $file (@files) {
    next if -l "$dir/$file" || -d _;
    $md5 .= Digest::MD5::md5_hex(readstr("$dir/$file"))."  $file\n";
  }
  $md5 = Digest::MD5::md5_hex($md5);
  return $md5;
}

sub getcode {
  my ($dir, $uri, $ineval) = @_;

  # evalize ourself
  if (!$ineval) {
    my $md5;
    eval {
     $md5 = getcode($dir, $uri, 1);
    };
    if ($@) {
      warn($@);
      return '';
    }
    return $md5;
  }

  my $ndir = "$dir.new";
  my $odir = "$dir.old";

  # clean up stale runs
  rm_rf($ndir) if -e $ndir;
  rm_rf($odir) if -e $odir;

  mkdir($ndir) || die("mkdir $ndir: $!\n");
  my $res;
  eval {
    $res = BSRPC::rpc({
      'uri' => $uri,
      'directory' => $ndir,
      'timeout' => $gettimeout,
      'withmd5' => 1,
      'createsubdirs' => 1,
      'receiver' => \&BSHTTP::cpio_receiver,
    });
  };
  die("getcode: $@") if $@;

  # got everything, clean things up, check if it really works
  if ($dir eq 'worker') {
    symlink('.', "$ndir/XML") || die("symlink: $!\n");
    chmod(0755, "$ndir/bs_worker");
    die("bs_worker selftest failed\n") if system("cd $ndir && ./bs_worker --selftest");
  } elsif ($dir eq 'build') {
    symlink('.', "$ndir/Date") || die("symlink: $!\n");
    symlink('.', "$ndir/Time") || die("symlink: $!\n");
    # we just change every file to be on the safe side
    chmod(0755, "$ndir/$_->{'name'}") for @$res;
  }

  # ok, commit
  if (-e $dir) {
    rename($dir, $odir) || die("rename $dir $odir: $!\n");
  }
  rename($ndir, $dir) || die("rename $ndir $dir: $!\n");
  rm_rf($odir) if -e $odir;
  my $md5 = '';
  for my $file (sort {$a->{'name'} cmp $b->{'name'}} @$res) {
    $md5 .= "$file->{'md5'}  $file->{'name'}\n" if $file->{'md5'};
  }
  $md5 = Digest::MD5::md5_hex($md5);
  return $md5;
}


sub rm_rf {
  my ($dir) = @_;
  BSUtil::cleandir($dir);
  rmdir($dir);
  die("rmdir $dir failed\n") if -d $dir;
}

sub importbuild {
    return if defined &Build::queryhdrmd5;
    unshift @INC, "$statedir/build";
    require Build;
    Build->import();
}

sub md5file {
  my ($fn) = @_;
  my $fd;
  open($fd, '<', $fn) || die("$fn: $!\n");
  my $ctx = Digest::MD5->new;
  $ctx->addfile($fd);
  close($fd);
  return $ctx->hexdigest();
}

sub sha256file {
  my ($fn) = @_;
  my $fd;
  open($fd, '<', $fn) || die("$fn: $!\n");
  my $ctx = Digest::SHA->new(256);
  $ctx->addfile($fd);
  close($fd);
  return $ctx->hexdigest();
}

sub getsources {
  my ($buildinfo, $dir) = @_;

  my @meta;
  push @meta, ($buildinfo->{'verifymd5'} || $buildinfo->{'srcmd5'})."  $buildinfo->{'package'}";
  my $server = $buildinfo->{'srcserver'} || $srcserver;

  my $res;
  eval {
    $res = BSRPC::rpc({
      'uri' => "$server/getsources",
      'directory' => $dir,
      'timeout' => $gettimeout,
      'withmd5' => 1,
      'receiver' => \&BSHTTP::cpio_receiver,
    }, undef, "project=$buildinfo->{'project'}", "package=$buildinfo->{'package'}", "srcmd5=$buildinfo->{'srcmd5'}");
  };
  die("getsources: $@") if $@;
  if (-e "$dir/.errors") {
    my $errors = readstr("$dir/.errors", 1);
    die("getsources: $errors");
  }
  # verify sources
  my %res = map {$_->{'name'} => $_} @$res;
  my $md5 = '';
  my @f = ls($dir);
  for my $f (sort @f) {
    die("unexpected file: $f") unless $res{$f};
    $md5 .= "$res{$f}->{'md5'}  $f\n";
  }
  $md5 = Digest::MD5::md5_hex($md5);
  die("source verification fails: $md5 != $buildinfo->{'verifymd5'}\n") if $md5 ne $buildinfo->{'verifymd5'};

  # add to slsa materials list
  if ($buildinfo->{'slsaprovenance'}) {
    add_slsa_materials_sourcefile($buildinfo, $_, "$dir/$_") for @f;
  }

  return @meta unless $buildinfo->{'file'} =~ /\.kiwi$/;

  # get additional kiwi sources
  my @sdep = grep {($_->{'repoarch'} || '') eq 'src'} @{$buildinfo->{'bdep'} || []};
  for my $src (@sdep) {
    die("additional kiwi sources not implemented for slsa provenance\n") if $buildinfo->{'slsaprovenance'};
    print "$src->{'name'}, ";
    my $idir = "$src->{'project'}/$src->{'package'}";
    $idir = "$dir/images/$idir";
    mkdir_p($idir);
    my $res = BSRPC::rpc({
      'uri' => "$server/getsources",
      'directory' => $idir,
      'timeout' => $gettimeout,
      'withmd5' => 1,
      'receiver' => \&BSHTTP::cpio_receiver,
    }, undef, "project=$src->{'project'}", "package=$src->{'package'}", "srcmd5=$src->{'srcmd5'}");
    if (-e "$idir/.errors") {
      my $errors = readstr("$idir/.errors", 1);
      die("getsources: $errors");
    }
    push @meta, "$src->{'srcmd5'}  $src->{'project'}/$src->{'package'}";
  }
  return @meta;
}

sub getjobdata {
  my ($buildinfo, $dir) = @_;

  my $server = $buildinfo->{'reposerver'};
  my $res = BSRPC::rpc({
    'uri' => "$server/getjobdata",
    'directory' => $dir,
    'timeout' => $gettimeout,
    'receiver' => \&BSHTTP::cpio_receiver,
  }, undef, "job=$buildinfo->{'job'}", "arch=$buildinfo->{'arch'}", "jobid=$buildinfo->{'jobid'}");
  return $res;
}

sub getfollowupsources {
  my ($buildinfo, $dir) = @_;
  my $server = $buildinfo->{'reposerver'};
  my $res = getjobdata($buildinfo, $dir);
  die("no old sources in followup job\n") unless @$res;
  # need to get buildenv from old sources ;(
  $server = $buildinfo->{'srcserver'} || $srcserver;
  $res = BSRPC::rpc({
    'uri' => "$server/source/$buildinfo->{'project'}/$buildinfo->{'package'}",
    'timeout' => $gettimeout,
  }, $BSXML::dir, "rev=$buildinfo->{'srcmd5'}");
  my %entries = map {$_->{'name'} => $_} @{$res->{'entry'} || []};
  my $bifile = "_buildenv.$buildinfo->{'repository'}.$buildinfo->{'arch'}";
  $bifile = '_buildenv' unless $entries{$bifile};
  return unless $entries{$bifile};
  BSRPC::rpc({
    'uri' => "$server/source/$buildinfo->{'project'}/$buildinfo->{'package'}/$bifile",
    'timeout' => $gettimeout,
    'receiver' => \&BSHTTP::file_receiver,
    'filename' => "$dir/_buildenv.$buildinfo->{'repository'}.$buildinfo->{'arch'}",
  }, undef, "rev=$buildinfo->{'srcmd5'}", 'noajax=1');
}

sub getobsgendiffdata {
  my ($buildinfo, $dir) = @_;
  my $obsgendiff = $buildinfo->{'obsgendiff'};
  return undef unless $obsgendiff;
  my $server = $buildinfo->{'srcserver'} || $srcserver;
  my $res = BSRPC::rpc({
    'uri' => "$server/getobsgendiffdata",
    'directory' => $dir,
    'timeout' => $gettimeout,
    'receiver' => \&BSHTTP::cpio_receiver,
  }, undef, "project=$obsgendiff->{'project'}", "repository=$obsgendiff->{'repository'}", "arch=$buildinfo->{'arch'}", "jobid=$buildinfo->{'jobid'}");
  return $res;
}

sub getmodulemddata {
  my ($buildinfo, $dir) = @_;
  my $server = $buildinfo->{'srcserver'} || $srcserver;
  my @args;
  push @args, map {"module=$_"} @{$buildinfo->{'module'} || []};
  push @args, "modularityplatform=$buildinfo->{'modularity_platform'}";
  push @args, "modularitylabel=$buildinfo->{'modularity_label'}";
  my $mds = BSRPC::rpc({
    'uri' => "$server/getmodulemd",
    'filename' => "$dir/_modulemd.in",
    'timeout' => $gettimeout,
  }, \&BSUtil::fromstorable, "project=$buildinfo->{'project'}", "package=$buildinfo->{'modularity_package'}", "srcmd5=$buildinfo->{'modularity_srcmd5'}", @args);
  $mds = [ $mds ] if ref($mds) eq 'HASH';
  for my $md (@$mds) {
    next unless $md->{'document'} eq 'modulemd';		# should do this in the src server
    $md->{'data'}->{'arch'} = $buildinfo->{'arch'};
  }
  BSUtil::store("$dir/_modulemd.pst", undef, $mds);
}

sub getpubkey {
  my ($buildinfo, $dir, $signtype) = @_;
  my $server = $buildinfo->{'srcserver'} || $srcserver;
  my @args = ("project=$buildinfo->{'project'}");
  push @args, "signtype=$signtype" if $signtype;
  push @args, "signflavor=$buildinfo->{'signflavor'}" if $buildinfo->{'signflavor'};
  my $pubkey = eval {
    BSRPC::rpc({
      'uri' => "$server/getpubkey",
      'timeout' => $gettimeout,
    }, undef, @args);
  };
  die("could not retrieve pubkey: $@\n") if $@;
  die("unusable pubkey\n") unless $pubkey && length($pubkey) > 10;
  writestr("$dir/_pubkey", undef, $pubkey);
}

sub getsslcert {
  my ($buildinfo, $dir, $signtype) = @_;
  my $server = $buildinfo->{'srcserver'} || $srcserver;
  my $cert;
  my @args = ("project=$buildinfo->{'project'}", "autoextend=1");
  push @args, "signtype=$signtype" if $signtype;
  push @args, "signflavor=$buildinfo->{'signflavor'}" if $buildinfo->{'signflavor'};
  eval {
    $cert = BSRPC::rpc({
      'uri' => "$server/getsslcert",
      'timeout' => $gettimeout,
    }, undef, @args);
  };
  die("could not retrieve ssl certificate: $@\n") if $@;
  writestr("$dir/_projectcert.crt", undef, $cert) if $cert;
}

sub qsystem {
  my (@args) = @_;

  my $pid;
  if (!($pid = xfork())) {
    $SIG{'PIPE'} = 'DEFAULT';
    open(STDOUT, ">/dev/null") if $silent;
    exec(@args);
    die("$args[0]: $!\n"); 
  }
  waitpid($pid, 0) == $pid || die("waitpid $pid: $!\n"); 
  return $?;
}

sub link_or_copy {
  my ($from, $to) = @_;
  unlink($to);
  return 1 if link($from, $to);
  local *F;
  local *G;
  return undef unless open(F, '<', $from);
  if (!open(G, '>', $to)) {
    close F;
    return undef;
  }
  my $buf;
  while (1) {
    my $r = sysread(F, $buf, 8192);
    if (!defined($r)) {
      warn("sysread $from: $!\n");
      unlink($to);
      close F;
      return undef;
    }
    last unless $r;
    if ((syswrite(G, $buf) || 0) != length($buf)) {
      warn("syswrite $to: $!\n");
      unlink($to);
      close F;
      return undef;
    }
  }
  close(F);
  if (!close(G)) {
    warn("close $to: $!\n");
    unlink($to);
    return undef;
  }
  return 1;
}

sub get_cachefile {
  my ($prpa, $key) = @_;
  return (undef, undef) unless defined $cachedir;
  my $cacheid = Digest::SHA::sha256_hex("$prpa/$key");
  my $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
  return ($cacheid, $cachefile);
}

sub move_entry_into_cache {
  my ($cacheid, $path) = @_;
  my $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
  mkdir_p("$cachedir/".substr($cacheid, 0, 2));
  unlink("$cachefile.$$");
  return 0 unless link_or_copy($path, "$cachefile.$$");
  rename("$cachefile.$$", $cachefile) || die("500 rename $cachefile.$$ $cachefile: $!\n");
  unlink("$cachefile.$$");
  my $mpath = "$path.meta";
  $mpath = "$1.meta" if $path =~ /^(.*)\.(?:$binsufsre)$/;
  if (-s $mpath) {
    unlink("$cachefile.meta.$$");
    if (link($mpath, "$cachefile.meta.$$")) {
      rename("$cachefile.meta.$$", "$cachefile.meta") || die("500 rename $cachefile.meta.$$ $cachefile.meta: $!\n");
    } else {
      unlink("$cachefile.meta");
    }
  } else {
    unlink("$cachefile.meta");
  }
  return 1;
}

sub remove_entry_from_cache {
  my ($cacheid) = @_;
  my $cachefile = "$cachedir/".substr($cacheid, 0, 2)."/$cacheid";
  unlink($cachefile);
  unlink("$cachefile.meta");
}

sub manage_cache {
  my ($prunesize, $cacheold, $cachenew) = @_;
  # get the lock
  local *F;
  BSUtil::lockopen(\*F, '+>>', "$cachedir/content", 1) || return;
  my $content;
  if (-s F) {
    seek(F, 0, 0);
    eval { $content = Storable::fd_retrieve(\*F) };
    die("FATAL: cannot read cache content: $@") if $@;
  }
  $content ||= [];
  my %content = map {$_->[0] => $_->[1]} @$content;
  # put cacheold, cachenew at the top
  splice(@$content, 0, 0, @$cacheold) if $cacheold && @$cacheold;
  if ($cachenew) {
    for my $c (reverse @$cachenew) {
      next unless move_entry_into_cache($c->[0], pop(@$c));
      unshift @$content, $c;
      $content{$c->[0]} = $c->[1];
    }
  }
  # prune cache
  my @pruneids;
  for my $c (@$content) {
    if (!defined delete $content{$c->[0]}) {
      $c = undef;	# already pruned
    } elsif (($prunesize -= $c->[1]) < 0) {
      push @pruneids, $c->[0];
      $c = undef;
    }
  }
  remove_entry_from_cache($_) for sort @pruneids;
  @$content = grep {defined $_} @$content;
  eval {
    Storable::nstore($content, "$cachedir/content.new");
    rename("$cachedir/content.new", "$cachedir/content") || die("500 rename $cachedir/content.new $cachedir/content: $!\n");
  };
  if ($@) {
    # could not update content, delete all new entries
    remove_entry_from_cache($_->[0]) for @$cachenew;
    die($@);
  }
  close F;
}

sub trygetbinariesproxy {
  my ($server, $param, $bins, $bvs, @args) = @_;

  return undef unless $getbinariesproxy;
  my @proxybins;
  for my $bin (@$bins) {
    my $bv = $bvs->{$bin};
    if (!$bv || !$bv->{'name'} || $bv->{'name'} !~ /\.($binsufsre)$/) {
      push @proxybins, '@'.$bin;
    } else {
      push @proxybins, $bv->{'hdrmd5'}.($bv->{'metamd5'} || '').':'.($bv->{'sizek'} || 0).":$1\@".$bin;
    }
  }
  my $res;
  my $callers_time = time();
  eval {
    $res = BSRPC::rpc({ %$param, 'uri' => "$getbinariesproxy/getbinaries" }, undef, "server=$server", @args, "now=$callers_time", 'binaries='.join(',', @proxybins));
  };
  if ($@) {
    warn($@);
    undef $res;
  }
  return $res;
}

sub trygetbinariesproxy_preinstallimage {
  my ($server, $param, $img) = @_;
  return undef unless $getbinariesproxy;
  my $res;
  my @args;
  push @args, "workerid=$workerid" if defined $workerid;
  push @args, "prpa=$img->{'prpa'}", "hdrmd5=$img->{'hdrmd5'}", "sizek=$img->{'sizek'}", "path=$img->{'path'}";
  my $callers_time = time();
  eval {
    $res = BSRPC::rpc({ %$param, 'uri' => "$getbinariesproxy/getpreinstallimage" }, undef, "server=$server", @args, "now=$callers_time");
  };
  if ($@) {
    warn($@);
    undef $res;
  }
  return $res;
}

sub checkmd5 {
  my ($file, $md5) = @_;
  local *F;
  open(F, '<', $file) || return undef;
  my $ctx = Digest::MD5->new;
  $ctx->addfile(*F);
  close F;
  return $ctx->hexdigest() eq $md5 ? 1 : undef;
}

sub checkbv {
  my ($file, $bv) = @_;
  if ($bv->{'hdrmd5'}) {
    my $leadsigmd5 = '';
    my $id = Build::queryhdrmd5($file, \$leadsigmd5) || '';
    return $id eq $bv->{'hdrmd5'} && (!$bv->{'leadsigmd5'} || $bv->{'leadsigmd5'} eq $leadsigmd5) ? 1 : 0;
  } elsif ($bv->{'md5sum'}) {
    return checkmd5($file, $bv->{'md5sum'});
  }
  return undef;	# can't check
}

sub getbinaries_cache {
  my ($dir, $server, $projid, $repoid, $arch, $nometa, $bins, $modules, $bvl, $binprefix) = @_;

  $binprefix ||= '';
  importbuild() unless defined &Build::queryhdrmd5;
  if (! -d $dir) {
    mkdir_p($dir) || die("mkdir_p $dir: $!\n");
  }
  my %ret;
  undef $bvl unless $cachedir;	# no use here if ther's no cachedir
  if ($bvl) {
    # check if it contains everything we want to know
    my %bv;
    for (@{$bvl->{'binary'} || []}) {
      if ($_->{'error'}) {
        $bv{$_->{'name'}} = 1;
      } else {
        $bv{$1} = 1 if $_->{'name'} =~ /(.*)\.(?:$binsufsre)$/;
      }
    }
    undef $bvl if grep {!$bv{$_}} @$bins;
  }
  if ($cachedir && !$bvl) {
    my $callers_time = time();
    my @args;
    push @args, "workerid=$workerid" if defined $workerid;
    push @args, "project=$projid";
    push @args, "repository=$repoid";
    push @args, "arch=$arch";
    push @args, "nometa" if $nometa;
    push @args, map {"module=$_"} @{$modules || []};
    push @args, "binaries=".join(',', @$bins);
    push @args, "now=$callers_time";
    push @args, "withevr=1";
    eval {
      $bvl = BSRPC::rpc({
        'uri' => "$server/getbinaryversions",
        'timeout' => $gettimeout,
        }, $BSXML::binaryversionlist, @args);
    };
    warn($@) if $@;
  }
  $bvl ||= {};
  my %bv;
  for (@{$bvl->{'binary'} || []}) {
    if ($_->{'error'}) {
      $bv{$_->{'name'}} = $_;
    } else {
      $bv{$1} = $_ if $_->{'name'} =~ /(.*)\.(?:$binsufsre)$/;
    }
  }
  my @downloadbins;
  my $downloadsizek = 0;
  my @cacheold;
  my @cachenew;
  for my $bin (@$bins) {
    my $bv = $bv{$bin};
    if (!$bv) {
      push @downloadbins, $bin;
      next;
    }
    next if $bv->{'error'};
    my ($cacheid, $cachefile) = get_cachefile("$projid/$repoid/$arch", $bv->{'hdrmd5'});
    if (!link_or_copy($cachefile, "$dir/$binprefix$bv->{'name'}")) {
      push @downloadbins, $bin;
      $downloadsizek += $bv->{'sizek'};
      next;
    }
    my $usecache = 1;
    my $havemeta;
    my @s = stat("$dir/$binprefix$bv->{'name'}");
    die unless @s;
    if (!$nometa && $bv->{'metamd5'}) {
      my $mn = $bv->{'name'};
      $mn =~ s/\.(?:$binsufsre)/.meta/;
      if (link_or_copy("$cachefile.meta", "$dir/$binprefix$mn")) {
	if (checkmd5("$dir/$binprefix$mn", $bv->{'metamd5'})) {
	  $havemeta = 1;
	} else {
	  unlink("$dir/$binprefix$mn");
	}
      }
      $usecache = 0 unless $havemeta;
    }
    if ($usecache) {
      # check hdrmd5 to be sure we got the right bin
      my $id = Build::queryhdrmd5("$dir/$binprefix$bv->{'name'}");
      $usecache = 0 if ($id || '') ne $bv->{'hdrmd5'};
    }
    if (!$usecache) {
      unlink("$dir/$binprefix$bv->{'name'}");
      push @downloadbins, $bin;
      $downloadsizek += $bv->{'sizek'};
    } else {
      push @cacheold, [$cacheid, $s[7]];
      my $r = { 'name' => "$binprefix$bv->{'name'}", 'hdrmd5' => $bv->{'hdrmd5'} };
      $r->{'meta'} = 1 if !$nometa && $havemeta;
      if ($bv->{'evr'}) {
	$r->{'evr'} = $bv->{'evr'};
	$r->{'arch'} = $bv->{'arch'};
      }
      $ret{"$binprefix$bin"} = $r;
    }
  }
  $binariescachehits += @cacheold;
  if (@downloadbins) {
    $binariesdownload += @downloadbins;
    $binariesdownloadsize += $downloadsizek;
    if ($cachedir && $downloadsizek * 1024 * 100 > $cachesize) {
      # reserve space
      manage_cache($cachesize - $downloadsizek * 1024);
    }
    my @args;
    push @args, "workerid=$workerid" if defined $workerid;
    push @args, "project=$projid";
    push @args, "repository=$repoid";
    push @args, "arch=$arch";
    push @args, map {"module=$_"} @{$modules || []};
    my $param = {
      'uri' => "$server/getbinaries",
      'directory' => $dir,
      'timeout' => $gettimeout,
      'map' => $binprefix,
      'receiver' => \&BSHTTP::cpio_receiver,
    };
    my $res;
    $res = trygetbinariesproxy($server, $param, \@downloadbins, \%bv, @args) if $getbinariesproxy;
    eval {
      my $callers_time = time();
      $res ||= BSRPC::rpc($param, undef, @args, "now=$callers_time", 'binaries='.join(',', @downloadbins));
    };
    # do not die if a (remote) project does not exist
    return {} if !@cacheold && $@ && $@ =~ /^404 /;
    die($@) if $@;
    die("Error\n") unless ref($res) eq 'ARRAY';
    my %havemeta;
    for my $r (@$res) {
      if ($r->{'name'} =~ /^(.*)\.(?:$binsufsre)$/) {
	my $n = $1;
	my @s = stat("$dir/$r->{'name'}");
	die unless @s;
	my $id = Build::queryhdrmd5("$dir/$r->{'name'}");
	if ($id) {
	  $r->{'hdrmd5'} = $id;
	  my ($cacheid) = get_cachefile("$projid/$repoid/$arch", $id);
	  push @cachenew, [$cacheid, $s[7], "$dir/$r->{'name'}"];
	}
	my $bv = $bv{substr($n, length($binprefix))};
	if ($id && $bv && $bv->{'evr'} && $bv->{'hdrmd5'} eq $id) {
	  $r->{'evr'} = $bv->{'evr'};
	  $r->{'arch'} = $bv->{'arch'};
	} else {
	  my $q = eval { Build::query("$dir/$r->{'name'}", 'evra' => 1) };
	  if ($q && $q->{'version'}) {
	    my $evr = $q->{'version'};
	    $evr .= "-$q->{'release'}" if defined $q->{'release'};
	    $r->{'evr'} = $q->{'epoch'} ? "$q->{'epoch'}:$evr" : $evr;
	    $r->{'arch'} = $q->{'arch'} if $q->{'arch'};
	  }
	}
	$ret{$n} = $r;
      } elsif ($r->{'name'} =~ /^(.*)\.meta$/) {
        $havemeta{$1} = 1;
      }
    }
    for (keys %havemeta) {
      next unless $ret{$_};
      $ret{$_}->{'meta'} = 1;
    }
  }
  manage_cache($cachesize, \@cacheold, \@cachenew) if $cachedir;
  if ($nometa) {
    for (keys %ret) {
      next unless $ret{$_}->{'meta'};
      unlink("$dir/$_.meta");
      delete $ret{$_}->{'meta'};
    }
  }
  return \%ret;
}

# bring xarch to the back... (last one wins)
sub prpasort {
  my ($xarch, @p) = @_;
  my %h;
  for my $prpa (@p) {
    my ($projid, $repoid, $arch) = split('/', $prpa, 3);
    $arch = "~$arch" if $arch eq $xarch;
    $h{$prpa} = "$projid/$repoid/$arch";
  }
  return sort { $h{$a} cmp $h{$b} } @p;
}

sub sendbadpackagebinaryversionlist {
  my ($server, $projid, $repoid, $arch, $packid) = @_;
  eval {
    my @args = ("project=$projid", "repository=$repoid", "arch=$arch", "package=$packid");
    push @args, "workerid=$workerid" if defined $workerid;
    BSRPC::rpc({
      'uri' => "$server/badpackagebinaryversionlist",
      'timeout' => $gettimeout,
    }, undef, @args);
  };
  warn($@) if $@;
}

# this specialized version of getbinaries only works for
# product builds, either done via kiwi or product composer
# TODO: add provenance materials
sub getbinaries_product {
  my ($buildinfo, $dir, $srcdir, $kiwiorigins) = @_;

  my $imagemode = getimagemode($buildinfo);
  mkdir_p($dir);
  # we need the Build package for queryhdrmd5
  importbuild() unless defined &Build::queryhdrmd5;

  # fetch packages needed for product building
  my @kdeps = map {$_->{'name'}} grep {!defined($_->{'package'})} @{$buildinfo->{'bdep'} || []};
  mkdir_p($dir);
  my $modules = $buildinfo->{'module'};
  for my $repo (@{$buildinfo->{'syspath'} || $buildinfo->{'path'} || []}) {
    last if !@kdeps;
    my $repoarch = $buildinfo->{'arch'};
    $repoarch = $BSConfig::localarch if $repoarch eq 'local' && $BSConfig::localarch;
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $got = getbinaries_cache($dir, $server, $repo->{'project'}, $repo->{'repository'}, $repoarch, 1, \@kdeps, $modules);
    @kdeps = grep {!$got->{$_}} @kdeps;
  }
  die("404: getbinaries_product: missing packages: @kdeps\n") if @kdeps;

  # all done for followup builds
  return () if $buildinfo->{'followupfile'};

  # now fetch all packages that go into the repos subdir

  my %prp2server;
  for (@{$buildinfo->{'path'} || []}) {
    $prp2server{"$_->{'project'}/$_->{'repository'}"} = $_->{'server'};
  }
  my %prpas;
  for my $dep (grep {defined($_->{'package'})} @{$buildinfo->{'bdep'} || []}) {
    my $repoarch = $dep->{'repoarch'} || $buildinfo->{'arch'};
    next if $repoarch eq 'src';
    $prpas{"$dep->{'project'}/$dep->{'repository'}/$repoarch"}->{$dep->{'package'}} = 1;
  }
  my @prpas = sort keys %prpas;
  if ($buildinfo->{'package'} =~ /-([^-]+)$/) {
    @prpas = prpasort($1, @prpas);
  }

  my %meta;
  my $rpmhdrs_only = $buildinfo->{'rpmhdrs_only'} ? "/rpmhdrs_only" : '';
  my $nodbgpkgs = $buildinfo->{'nodbgpkgs'};
  my $nosrcpkgs = $buildinfo->{'nosrcpkgs'};

  my %cachenew;
  my @cacheold;
  my $downloadsizek = 0;
  my %chksumfiles;

  for my $prpa (@prpas) {
    my ($projid, $repoid, $arch) = split('/', $prpa, 3);
    my $prp = "$projid/$repoid";
    my $server =  $prp2server{$prp} || $buildinfo->{'reposerver'};
    my $ddir = "$srcdir/repos/$prp";
    my $needchksums;

    # setup binary filter for this prpa
    my %binfilter;
    for my $dep (@{$buildinfo->{'bdep'} || []}) {
      next unless defined($dep->{'package'}) && $dep->{'project'} eq $projid && $dep->{'repository'} eq $repoid && ($dep->{'repoarch'} || $dep->{'arch'}) eq $arch;
      if (($dep->{'binary'} || '') =~ /^(::import::.*?::.*)-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.d?rpm$/s) {
	$binfilter{"$dep->{'package'}/$1.$2"} = 1;
      } else {
	$binfilter{"$dep->{'package'}/$dep->{'name'}.$dep->{'arch'}"} = 1;
      }
    }

    # create rarch calculation closure function
    my $calc_rarch = sub {
      my ($name, $packid) = @_;
      if ($name =~ /^((?:::import::.*::)?.*)-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.d?rpm$/) {
	return $binfilter{"$packid/$1.$2"} ? $2 : undef;
      } elsif ($name =~ /^((?:::import::.*::)?.*)-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.slsa_provenance\.json$/) {
	return $binfilter{"$packid/$1.$2"} ? $2 : undef;
      } elsif ($name eq '_modulemd.yaml') {
	return 'modulemd';
      } elsif ($name eq '_slsa_provenance.json') {
	return '_slsa_provenance';
      } elsif ($name =~ /appdata\.xml$/) {
	return 'appdata'; 
      } elsif ($name eq 'updateinfo.xml') {
	return 'updateinfo';
      }
      return undef;
    };

    # get the bininfo of the prpa if we need it
    my $pbvl;
    if ($cachedir || $getbinariesproxy_kiwiproduct) {
      my @prpapackages = sort keys %{$prpas{$prpa}};
      eval {
	if ($server ne $buildinfo->{'srcserver'}) {
	  $pbvl = BSRPC::rpc({
	    'uri' => "$server/getpackagebinaryversionlist",
	    'request' => 'POST',
	    'timeout' => $gettimeout,
	  }, $BSXML::packagebinaryversionlist, "project=$projid", "repository=$repoid", "arch=$arch", map {"package=$_"} @prpapackages);
	} else {
	  $pbvl = BSRPC::rpc({
	    'uri' => "$server/build/$prpa",
	    'timeout' => $gettimeout,
	  }, $BSXML::packagebinaryversionlist, "view=binaryversions", map {"package=$_"} @prpapackages);
	}
      };
      undef $pbvl if $@;
      warn($@) if $@;
      $pbvl = { map {$_->{'package'} => $_} @{$pbvl->{'binaryversionlist'} || []} } if $pbvl;
    }
    $pbvl ||= {};

    for my $packid (sort keys %{$prpas{$prpa}}) {
      mkdir_p($ddir);

      # get the bininfo of the package if we need it and we do not have the pbvl entry
      my $bvl = $pbvl->{$packid};
      if (!$bvl && ($cachedir || $getbinariesproxy_kiwiproduct)) {
	eval {
	  $bvl = BSRPC::rpc({
	    'uri' => "$server/build/$prpa/$packid",
	    'timeout' => $gettimeout,
	  }, $BSXML::binaryversionlist, 'view=binaryversions');
	};
	undef $bvl if $@;
	warn($@) if $@;
      }

      my @todo;
      my @done;
      my %knownmd5;
      if ($bvl) {
	for my $bv (@{$bvl->{'binary'} || []}) {
	  my $bin = $bv->{'name'};
	  my $rarch = $calc_rarch->($bin, $packid);
	  next unless $rarch;
	  next if $nosrcpkgs && ($rarch eq 'src' || $rarch eq 'nosrc');
	  next if $nodbgpkgs && $bin =~ /-(?:debuginfo|debugsource)-/;
	  $bv->{'rarch'} = $rarch;
	  push @todo, $bv;
	}
      }

      # try the cache first
      if (@todo && $cachedir) {
	for my $bv (splice @todo) {
	  my $cachekey;
	  if ($bv->{'hdrmd5'}) {
	    $cachekey = "$bv->{'hdrmd5'}$rpmhdrs_only";
	  } elsif ($bv->{'md5sum'}) {
	    $cachekey = "$bv->{'md5sum'}.extra";
	  }
	  if ($cachekey) {
	    my ($cacheid, $cachefile) = get_cachefile($prpa, $cachekey);
	    my $rarch = $bv->{'rarch'};
	    my $filename = "$rarch/$bv->{'name'}";
	    delete $cachenew{"$ddir/$filename"};
	    mkdir_p("$ddir/$rarch");
	    if (link_or_copy($cachefile, "$ddir/$filename.new.rpm")) {
	      my @s = stat("$ddir/$filename.new.rpm");
	      if (@s && checkbv("$ddir/$filename.new.rpm", $bv)) {
		push @cacheold, [$cacheid, $s[7]];
		$knownmd5{$filename} = $bv->{'hdrmd5'} if $bv->{'hdrmd5'};
		push @done, $filename;
		next;
	      }
	      unlink("$ddir/$filename.new.rpm");
	    }
	  }
	  push @todo, $bv;
	}
      }

      #print "(cache: ".@done." hits, ".@todo." misses)";
      $binariesdownload += @todo;
      $binariescachehits += @done;

      # reserve space
      $downloadsizek += $_->{'sizek'} || 0 for @todo;
      if (@todo && $cachedir && $downloadsizek * 1024 * 100 > $cachesize) {
	manage_cache($cachesize - $downloadsizek * 1024, \@cacheold, [ values %cachenew ]);
	@cacheold = ();
	%cachenew = ();
	$binariesdownloadsize += $downloadsizek;
	$downloadsizek = 0;
      }

      # download individual missing binaries from the getbinariesproxy
      if (@todo && $getbinariesproxy_kiwiproduct && !$rpmhdrs_only) {
	my @args;
	push @args, "workerid=$workerid" if defined $workerid;
	push @args, "noajax=1" if $server ne $buildinfo->{'srcserver'};
	my %tofetch;
	for my $bv (@todo) {
	  my $sizek = $bv->{'sizek'} || 0;
	  if ($bv->{'name'} =~ /\.d?rpm$/) {
	    next unless $bv->{'hdrmd5'};
	    push @args, "binary=$bv->{'hdrmd5'}:$sizek:rpm\@$bv->{'name'}";
	  } else {
	    next unless $bv->{'md5sum'};
	    push @args, "binary=$bv->{'md5sum'}:$sizek:extra\@$bv->{'name'}";
	  }
	  $tofetch{$bv->{'name'}} = [ $bv ];
	}
	if (%tofetch) {
	  my $param = {
	    'uri' => "$getbinariesproxy_kiwiproduct/getbinaries_kiwiproduct",
	    'request' => 'POST',
	    'directory' => $ddir,
	    'timeout' => $gettimeout,
	    'map' => sub {
	      my ($param, $name) = @_;
	      return undef unless $tofetch{$name};
	      my $rarch = $tofetch{$name}->[0]->{'rarch'};
	      $tofetch{$name}->[1] = "$rarch/$name";
	      mkdir_p("$param->{'directory'}/$rarch");
	      return "$rarch/$name.new.rpm";
	    },
	    'receiver' => \&BSHTTP::cpio_receiver,
	  };
	  my $callers_time = time();
	  eval { BSRPC::rpc($param, undef, "project=$projid", "repository=$repoid", "arch=$arch", "package=$packid", "server=$server", @args, "now=$callers_time") };
	  if ($@) {
	    warn($@);
	    for (values %tofetch) {
	      unlink("$ddir/$_->[1].new.rpm") if $_->[1];
	    }
	    %tofetch = ();
	  }
	  # verify everything we fetched (we need to make sure the leadsigmd5 matches!)
	  for my $bv (splice @todo) {
	    my $bin = $bv->{'name'};
	    my $filename = ($tofetch{$bin} || [])->[1];
	    if (!$filename) {
	      push @todo, $bv;
	      next;
	    }
	    my @s = stat("$ddir/$filename.new.rpm");
	    if (@s && checkbv("$ddir/$filename.new.rpm", $bv)) {
	      my $cachekey = $bv->{'hdrmd5'} ? "$bv->{'hdrmd5'}$rpmhdrs_only" : "$bv->{'md5sum'}.extra";
	      my ($cacheid) = get_cachefile($prpa, $cachekey);
	      $cachenew{"$ddir/$filename"} = [$cacheid, $s[7], "$ddir/$filename"];
	      push @done, $filename;
	      $knownmd5{$filename} = $bv->{'hdrmd5'} if $bv->{'hdrmd5'};
	    } else {
	      delete $cachenew{"$ddir/$filename"};
	      unlink("$ddir/$filename.new.rpm");	# file does not match, delete
	      push @todo, $bv;
	    }
	  }
	}
      }

      # download individual missing binaries from the server
      if (@todo) {
	my %tofetch = map {$_->{'name'} => $_} @todo;
	eval {
	  my @todo_chunked = @todo;
	  while (@todo_chunked) {
	    my @args;
	    push @args, $rpmhdrs_only ? "view=cpioheaderchksums" : "view=cpio";
	    push @args, "noajax=1" if $server ne $buildinfo->{'srcserver'};
	    push @args, map {"binary=$_->{'name'}"} splice(@todo_chunked, 0, 50);
	    my $param = {
	      'uri' => "$server/build/$prpa/$packid",
	      'directory' => $ddir,
	      'timeout' => $gettimeout,
	      'map' => sub {
		my ($param, $name) = @_;
		my $rarch = ($tofetch{$name} || {})->{'rarch'};
		return undef unless $rarch;
		mkdir_p("$ddir/$rarch");
		return "$rarch/$name.new.rpm";
	      },
	      'receiver' => \&BSHTTP::cpio_receiver,
	    };
	    BSRPC::rpc($param, undef, @args);
	  }
	};
	if ($@) {
	  # delete everything as a file might be truncated
	  unlink("$ddir/$_->{'rarch'}/$_->{'name'}.new.rpm") for @todo;
	}
	# verify the downloaded files
	for my $bv (splice @todo) {
	  my $filename = "$bv->{'rarch'}/$bv->{'name'}";
	  delete $cachenew{"$ddir/$filename"};
	  my @s = stat("$ddir/$filename.new.rpm");
	  if (!@s) {
	    push @todo, $bv;
	    next;
	  }
	  if (!$bv->{'hdrmd5'} && !$bv->{'md5sum'}) {
	    # we cannot verify the file. we could put it in the cache,
	    # but it is likely that we will also not get a cachekey
	    # in the future.
	  } elsif (checkbv("$ddir/$filename.new.rpm", $bv)) {
	    # we got the right file, put it in the cache
	    my $cachekey = $bv->{'hdrmd5'} ? "$bv->{'hdrmd5'}$rpmhdrs_only" : "$bv->{'md5sum'}.extra";
	    my ($cacheid) = get_cachefile($prpa, $cachekey);
	    $cachenew{"$ddir/$filename"} = [$cacheid, $s[7], "$ddir/$filename"];
	  } else {
	    # we got a different file. too dangerous to take it, so fetch the complete package
	    unlink("$ddir/$filename.new.rpm");
	    push @todo, $bv;
	    next;
	  }
	  push @done, $filename;
	  $knownmd5{$filename} = $bv->{'hdrmd5'} if $bv->{'hdrmd5'};
	}
	#print "(still ".@todo." misses)";
      }

      if ($bvl && !@todo) {
	# got all entries, commit
	for (@done) {
	  rename("$ddir/$_.new.rpm", "$ddir/$_") || die("rename $ddir/$_.new.rpm $ddir/$_: $!\n");
	}
      }

      # if the above failed download the complete package
      if (!$bvl || @todo) {
	$binariesdownload -= @done + @todo;
	for (@done) {
	  unlink("$ddir/$_.new.rpm");
	  delete $cachenew{"$ddir/$_"};
	}
	@done = ();
	%knownmd5 = ();
	my @args;
	push @args, $rpmhdrs_only ? "view=cpioheaderchksums" : "view=cpio";
	push @args, "noajax=1" if $server ne $buildinfo->{'srcserver'};
	my $res = BSRPC::rpc({
	  'uri' => "$server/build/$prpa/$packid",
	  'directory' => $ddir,
	  'timeout' => $gettimeout,
	  'map' => sub {
	    my ($param, $name) = @_;
	    my $rarch = $calc_rarch->($name, $packid);
	    return undef unless $rarch;
	    return undef if $nodbgpkgs && $name =~ /-(?:debuginfo|debugsource)-/;
	    return undef if $nosrcpkgs && ($rarch eq 'src' || $rarch eq 'nosrc');
	    mkdir_p("$ddir/$rarch");
	    return "$rarch/$name";
	  },
	  'receiver' => \&BSHTTP::cpio_receiver,
	}, undef, @args);
	@done = map {$_->{'name'}} @$res;
	$binariesdownload += @done;
	$binariesdownloadsize += int($_->{'size'} / 1024) for @$res;

	# put into our cache
	if ($cachedir) {
	  for my $filename (@done) {
	    my @s = stat("$ddir/$filename");
	    next unless @s;
	    if ($filename !~ /\.rpm$/) {
	      # we could add the file to the cache, but it is not
	      # clear if we ever get a md5sum for it in the bv.
	      # so just ignore
	      next;
	    }
	    my $id = Build::queryhdrmd5("$ddir/$filename");
	    next unless $id;
	    my ($cacheid) = get_cachefile($prpa, "$id$rpmhdrs_only");
	    $cachenew{"$ddir/$filename"} = [$cacheid, $s[7], "$ddir/$filename"];
	    $knownmd5{$filename} = $id;
	  }
	  #print "(put ".@cachenew." entries)";
	}
	# the bvl entry seems to be outdated. tell our server about that.
	sendbadpackagebinaryversionlist($server, $projid, $repoid, $arch, $packid) if $bvl && $server ne $buildinfo->{'srcserver'};
      }

      # postprocess
      my %done = map {$_ => 1} @done;
      my $prpap = "$prpa/$packid";
      for (@done) {
	$kiwiorigins->{"obs://$prp/$_"} = $prpap;
      }

      # rename modulemd.yaml file
      if ($done{'modulemd/_modulemd.yaml'}) {
	my $md5 = md5file("$ddir/modulemd/_modulemd.yaml");
	$meta{"$prpap/_modulemd.yaml"} = "$md5  $prpap/_modulemd.yaml" if ($imagemode || '') eq 'productcompose';
	rename("$ddir/modulemd/_modulemd.yaml", "$ddir/modulemd/$packid.$arch.modulemd.yaml") || die("rename $ddir/modulemd/_modulemd.yaml $ddir/modulemd/$packid.$arch.modulemd.yaml: $!\n");
	$kiwiorigins->{"obs://$prp/modulemd/$packid.$arch.modulemd.yaml"} = $prpap;
	delete $kiwiorigins->{"obs://$prp/modulemd/_modulemd.yaml"};
	my $cn = delete $cachenew{"$ddir/modulemd/_modulemd.yaml"};
	if ($cn) {
	  $cn->[2] = "$ddir/modulemd/$packid.$arch.modulemd.yaml";
	  $cachenew{"$ddir/modulemd/$packid.$arch.modulemd.yaml"} = $cn;
	}
      }

      # rename updateinfo.xml file
      if ($done{'updateinfo/updateinfo.xml'}) {
	my $md5 = md5file("$ddir/updateinfo/updateinfo.xml");
	$meta{"$prpap/updateinfo.xml"} = "$md5  $prpap/updateinfo.xml" if ($imagemode || '') eq 'productcompose';
	rename("$ddir/updateinfo/updateinfo.xml", "$ddir/updateinfo/$packid.$arch.updateinfo.xml") || die("rename $ddir/updateinfo/updateinfo.xml $ddir/updateinfo/$packid.$arch.updateinfo.xml: $!\n");
	$kiwiorigins->{"obs://$prp/updateinfo/$packid.$arch.updateinfo.xml"} = $prpap;
	delete $kiwiorigins->{"obs://$prp/updateinfo/updateinfo.xml"};
	my $cn = delete $cachenew{"$ddir/updateinfo/updateinfo.xml"};
	if ($cn) {
	  $cn->[2] = "$ddir/updateinfo/$packid.$arch.updateinfo.xml";
	  $cachenew{"$ddir/updateinfo/$packid.$arch.updateinfo.xml"} = $cn;
	}
      }

      # spread _slsa_provenance.json to individual rpms
      if ($done{'_slsa_provenance/_slsa_provenance.json'}) {
	for my $rpm (sort keys %done) {
	  next unless $rpm =~ /(.*)\.rpm$/;
	  next if $rpm =~ /\/::import::/;
	  next if $done{"$1.slsa_provenance.json"};
	  unlink("$ddir/$1.slsa_provenance.json");
	  link("$ddir/_slsa_provenance/_slsa_provenance.json", "$ddir/$1.slsa_provenance.json") || die("link $ddir/_slsa_provenance/_slsa_provenance.json $ddir/$1.slsa_provenance.json: $!\n");
	  $kiwiorigins->{"obs://$prp/$1.slsa_provenance.json"} = $prpap;
	  my $cn = delete $cachenew{"$ddir/_slsa_provenance/_slsa_provenance.json"};
	  if ($cn) {
	    $cn->[2] = "$ddir/$1.slsa_provenance.json";
	    $cachenew{"$ddir/$1.slsa_provenance.json"} = $cn;
	  }
	}
	delete $cachenew{"$ddir/_slsa_provenance/_slsa_provenance.json"};
	unlink("$ddir/_slsa_provenance/_slsa_provenance.json");
	rmdir("$ddir/_slsa_provenance");
	delete $kiwiorigins->{"obs://$prp/_slsa_provenance/_slsa_provenance.json"};
      }

      # add all rpm entries to meta hash
      for my $f (@done) {
	my (undef, $name) = split('/', $f, 2);
	next unless defined($name) && $name =~ /^(?:::import::.*::)?(.*)-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.d?rpm$/;
	my ($n, $rarch) = ($1, $2);
	my $id = $knownmd5{$f};
	$id ||= Build::queryhdrmd5("$ddir/$f");
	$id ||= 'deaddeaddeaddeaddeaddeaddeaddead';
	$meta{"$prpap/$name"} = "$id  $prpap/$n.$rarch";
        $needchksums = 1;
      }
    }

    if ($needchksums) {
      # get checksums file
      eval {
	if ($server eq $buildinfo->{'srcserver'}) {
	  my $chk = BSRPC::rpc({
	    'uri' => "$server/build/$prpa",
	    'timeout' => $gettimeout,
	  }, $BSXML::packagebinarychecksums, 'view=binarychecksums');
	  $chk = { map { $_->{'package'} => $_->{'_content'} } @{$chk->{'binarychecksums'} || []} };
	  BSUtil::store("$ddir/.checksums.$arch", undef, $chk);
	} else {
	  BSRPC::rpc({
	    'uri' => "$server/build/$prpa",
	    'timeout' => $gettimeout,
	    'receiver' => \&BSHTTP::file_receiver,
	    'filename' => "$ddir/.checksums.$arch",
	  }, undef, 'view=rawbinarychecksums');
	}
	$chksumfiles{$prpa} = "$ddir/.checksums.$arch" if -s "$ddir/.checksums.$arch";
      };
    }
  }
  $binariesdownloadsize += $downloadsizek;

  manage_cache($cachesize, \@cacheold, [ values %cachenew ]) if $cachedir;

  # merge all the checksum files we downloaded
  unlink("$srcdir/repos/.createrepo_checksums");
  if (%chksumfiles) {
    if (open(CHKSUMS, '>', "$srcdir/repos/.createrepo_checksums")) {
      for my $chksumfile (sort keys %chksumfiles) {
	my $chksums = BSUtil::retrieve($chksumfile, 1) || {};
        for my $packid (sort keys %$chksums) {
          print CHKSUMS $chksums->{$packid};
	}
      }
      close CHKSUMS;
    }
    unlink("$srcdir/repos/.createrepo_checksums") unless -s "$srcdir/repos/.createrepo_checksums";
  }

  # create meta
  my @meta = map {$meta{$_}} sort {substr($meta{$a}, 34) cmp substr($meta{$b}, 34) || $a cmp $b} keys %meta;
  return BSUtil::unify(@meta);
}

sub getpreinstallimage_metas {
  my ($buildinfo, $dir, $metas, $img, $hdrmd5s) = @_;

  my $prpa = "$buildinfo->{'project'}/$buildinfo->{'repository'}/$buildinfo->{'arch'}";

  # find out which metas are needed for this image
  my @bins;
  my %havehdrmd5 = map {$_ => 1} @{$img->{'hdrmd5s'} || []};
  for my $bin (sort(keys %$metas)) {
    push @bins, $bin if $hdrmd5s->{$bin} && $havehdrmd5{$hdrmd5s->{$bin}};
  }

  my @todo;
  if ($cachedir) {
    # check the cache
    for my $bin (@bins) {
      my $bv = $metas->{$bin};
      my ($cacheid, $cachefile) = get_cachefile($prpa, $bv->{'hdrmd5'});
      if (link_or_copy("$cachefile.meta", "$dir/$bin.meta")) {
	next if checkmd5("$dir/$bin.meta", $bv->{'metamd5'});
	unlink("$dir/$bin.meta");
      }
      push @todo, $bin;
    }
  } else {
    @todo = @bins;
  }
  return 1 unless @todo;

  # get missing ones for the repo server for this build
  my @args;
  push @args, "workerid=$workerid" if defined $workerid;
  push @args, "project=$buildinfo->{'project'}";
  push @args, "repository=$buildinfo->{'repository'}";
  push @args, "arch=$buildinfo->{'arch'}";
  push @args, "metaonly=1";
  my $repo = (grep {$_->{'project'} eq $buildinfo->{'project'} && $_->{'repository'} eq $buildinfo->{'repository'}} @{$buildinfo->{'path'} || []})[0] || {};
  my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
  my $param = {
    'uri' => "$server/getbinaries",
    'directory' => $dir,
    'timeout' => $gettimeout,
    'receiver' => \&BSHTTP::cpio_receiver,
  };
  my $res;
  $res = trygetbinariesproxy($server, $param, \@todo, $metas, @args) if $getbinariesproxy;
  eval {
    my $callers_time = time();
    $res ||= BSRPC::rpc($param, undef, @args, "now=$callers_time", "binaries=".join(',', @todo));
    die("Error\n") unless ref($res);
  };
  warn($@) if $@;
  return undef unless $res;
  my %havemeta;
  for my $r (@$res) {
    next unless $r->{'name'} =~ /^(.*)\.meta$/;
    my $bin = $1;
    my $bv = $metas->{$bin};
    die("downloaded the wrong meta\n") unless $bv;
    local *F;
    next unless open(F, '<', "$dir/$bin.meta");
    my $ctx = Digest::MD5->new;
    $ctx->addfile(*F);
    close F;
    if ($ctx->hexdigest() eq $bv->{'metamd5'}) {
      # matching meta!
      $havemeta{$bin} = 1;
      if ($cachedir) {
	# put in cache so that we don't need to download it the next time
	my ($cacheid, $cachefile) = get_cachefile($prpa, $bv->{'hdrmd5'});
	unlink("$cachefile.meta.$$");
	if (link_or_copy("$dir/$bin.meta", "$cachefile.meta.$$")) {
	  rename("$cachefile.meta.$$", "$cachefile.meta") || die("rename $cachefile.meta.$$ $cachefile.meta: $!\n");
	}
      }
    } else {
      unlink("$dir/$bin.meta");
    }
  }
  return (grep {!$havemeta{$_}} @todo) ? undef : 1;
}

sub getpreinstallimage {
  my ($buildinfo, $dir, $todo, $bvls) = @_;

  return undef unless $Build::Features::preinstallimage;
  return undef if $buildinfo->{'slsaprovenance'};	# for now
  # get all bvls to calculate image data
  my @bins = @$todo;
  my %prpas;	# server -> prpas on this server
  my %targetbvs;
  my %imageorigins;
  my $projid = $buildinfo->{'project'};
  my $repoid = $buildinfo->{'repository'};
  my %metas;
  my $arch = $buildinfo->{'crossarch'} || $buildinfo->{'arch'};
  for my $repo (@{$buildinfo->{'path'} || []}) {
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $nometa = $repo->{'project'} ne $projid || $repo->{'repository'} ne $repoid ? 1 : 0;
    $nometa = 1 if $buildinfo->{'forcebinaryidmeta'};
    $nometa = 1 if $buildinfo->{'crossarch'};	# preinstallimage is always native
    my $callers_time = time();
    my @args;
    push @args, "workerid=$workerid" if defined $workerid;
    push @args, "project=$repo->{'project'}";
    push @args, "repository=$repo->{'repository'}";
    push @args, "arch=$arch";
    push @args, "nometa" if $nometa;
    push @args, map {"module=$_"} @{$buildinfo->{'module'} || []};
    push @args, "binaries=".join(',', @bins);
    push @args, "now=$callers_time";
    push @args, "withevr=1";
    my $bvl;
    eval {
      $bvl = BSRPC::rpc({
	'uri' => "$server/getbinaryversions",
	'timeout' => $gettimeout,
      }, $BSXML::binaryversionlist, @args);
    };
    warn($@) if $@;
    last unless $bvl;	# stop here
    push @{$prpas{$server}}, "$repo->{'project'}/$repo->{'repository'}/$arch" if $server ne $buildinfo->{'srcserver'};
    $bvls->{"$repo->{'project'}/$repo->{'repository'}/$arch"} = $bvl if $bvls;
    my %bv;
    for (@{$bvl->{'binary'} || []}) {
      if ($_->{'error'}) {
	$bv{$_->{'name'}} = $_;
      } else {
	next unless $_->{'name'} =~ /(.*)\.(?:$binsufsre)$/;
	$bv{$1} = $_;
	$metas{$1} = $_ if !$nometa && $_->{'metamd5'};
      }
    }
    my @origin = ($repo->{'project'}, $repo->{'repository'}, $arch);
    for my $bin (@bins) {
      my $bv = $bv{$bin} || {};
      next if $bv->{'error'};
      $targetbvs{$bin} = $bv;
      $imageorigins{$bin} = \@origin if $bv->{'hdrmd5'};
    }
    @bins = grep {!$targetbvs{$_}} @bins;
    last unless @bins;
  }
  # wipe error packages from targetbvs
  for (keys %targetbvs) {
    delete $targetbvs{$_} unless $targetbvs{$_}->{'hdrmd5'};
  }
  # wipe noinstall packages from hdrmd5s (buildengine case)
  for (grep {$_->{'noinstall'}} @{$buildinfo->{'bdep'} || []}) {
    delete $targetbvs{$_->{'name'}};
  }
  return undef unless %targetbvs && %prpas;

  # ok, now check if there is an image on one of the repo servers
  my %targethdrmd5s = %targetbvs;
  $_ = $_->{'hdrmd5'} for values %targethdrmd5s;
  my $match = "\0" x 512;
  for (values %targethdrmd5s) {
    vec($match, hex(substr($_, 0, 3)), 1) = 1;
  }
  #$match = unpack("H*", $match);	# hexify
  my @images;
  for my $server (sort(keys %prpas)) {
    my $serverimages;
    eval { 
      $serverimages = BSRPC::rpc({
	'uri' => "$server/getpreinstallimageinfos",
	'request' => 'POST',
	'data' => $match,
	'headers' => [ 'Content-Type: application/octet-stream' ],
	'timeout' => $gettimeout,
      }, undef, "match=body", map {"prpa=$_"} @{$prpas{$server}});
    };
    warn($@) if $@;
    if ($serverimages && substr($serverimages, 0, 4) eq 'pst0') {
      $serverimages = BSUtil::fromstorable($serverimages, 2);
    }
    $serverimages = [] unless $serverimages && ref($serverimages);
    $_->{'server'} = $server for @$serverimages;
    push @images, @$serverimages;
  }

  # got all available images, now find a good one to use
  my $imagefile;
  my $imageinfo;
  my %neededhdrmd5s = map {$_ => 1} values %targethdrmd5s;
  while (@images) {
    my $bestimgn = 2;
    my $bestimg;
    for my $img (@images) {
      # check if it really fits!
      next if @{$img->{'hdrmd5s'} || []} < $bestimgn;
      next unless $img->{'sizek'} && $img->{'hdrmd5'};
      next if grep {!$neededhdrmd5s{$_}} @{$img->{'hdrmd5s'} || []};
      # ignore our own image
      next if $img->{'prpa'} eq "$projid/$repoid/$arch" && $img->{'package'} eq $buildinfo->{'package'};
      if ($buildinfo->{'file'} eq '_preinstallimage') {
	# for building preinstall images we want at least one new package to avoid cycles
	my %havehdrmd5 = map {$_ => 1} @{$img->{'hdrmd5s'} || []};
	next unless grep {!$havehdrmd5{$_}} values %targethdrmd5s;
      }
      $bestimg = $img;
      $bestimgn = @{$img->{'hdrmd5s'} || []};
    }
    last unless $bestimg;	# nothing fits
    # try this one
    my $ifile = "preinstallimage.$bestimg->{'file'}";
    unlink("$dir/$ifile");

    # check if the image is in the cache
    my ($cacheid, $cachefile);
    if ($cachedir) {
      ($cacheid, $cachefile) = get_cachefile($bestimg->{'prpa'}, $bestimg->{'hdrmd5'});
      my $cachefilemeta = readstr("$cachefile.meta", 1) || '';
      if ($cachefilemeta eq "$bestimg->{'hdrmd5'}  :preinstallimage\n") {
	if (link_or_copy($cachefile, "$dir/$ifile")) {
	  # re-check to make races unlikely
	  $cachefilemeta = readstr("$cachefile.meta", 1) || '';
	  if ($cachefilemeta eq "$bestimg->{'hdrmd5'}  :preinstallimage\n") {
	    # looks good, use it!
	    my @s = stat("$dir/$ifile");
	    if (@s) {
	      # put entry on top
	      manage_cache($cachesize, [ [$cacheid, $s[7]] ], undef);
	    }
	    if (!%metas || getpreinstallimage_metas($buildinfo, $dir, \%metas, $bestimg, \%targethdrmd5s)) {
	      $imagefile = $ifile;
	      $imageinfo = $bestimg;
	      last;
	    }
	  }
	}
	unlink("$dir/$ifile");
      }
    }

    # nope, download it
    manage_cache($cachesize - $bestimg->{'sizek'} * 1024) if $cachedir;	# make room
    my $param = {
      'uri' => "$bestimg->{'server'}/build/$bestimg->{'prpa'}/$bestimg->{'path'}",
      'timeout' => $gettimeout,
      'receiver' => \&BSHTTP::file_receiver,
      'filename' => "$dir/$ifile",
    };
    my $res;
    $res = trygetbinariesproxy_preinstallimage($bestimg->{'server'}, $param, $bestimg) if $getbinariesproxy;
    eval { $res ||= BSRPC::rpc($param) };
    if ($@) {
      warn($@);
    } elsif (-s "$dir/$ifile") {
      my @s = stat(_);
      if ($cachedir && @s) {
	# put in cache (and fake meta)
	writestr("$dir/$ifile.meta", undef, "$bestimg->{'hdrmd5'}  :preinstallimage\n");
	manage_cache($cachesize, undef, [ [$cacheid, $s[7], "$dir/$ifile"] ]);
	unlink("$dir/$ifile.meta");
      }
      if (!%metas || getpreinstallimage_metas($buildinfo, $dir, \%metas, $bestimg, \%targethdrmd5s)) {
        $imagefile = $ifile;
        $imageinfo = $bestimg;
        last;
      }
    }
    unlink("$dir/$ifile");

    # that did not turn out well, try another image
    $bestimg->{'hdrmd5s'} = [];	# disables this image
  }

  return undef unless $imagefile;

  my %imagebins;
  my %havehdrmd5 = map {$_ => 1} @{$imageinfo->{'hdrmd5s'} || []};
  for my $bin (keys %targetbvs) {
    $imagebins{$bin} = $targetbvs{$bin} if $havehdrmd5{$targetbvs{$bin}->{'hdrmd5'}};
  }
  for my $bin (keys %metas) {
    delete $metas{$bin} unless $imagebins{$bin};	# only return metas on the image
  }
  for my $bin (keys %imageorigins) {
    delete $imageorigins{$bin} unless $imagebins{$bin};	# only return origins on the image
  }
  my $imagesource = $imageinfo->{'prpa'};
  $imagesource =~ s/\/[^\/]*$//;	# strip arch
  $imagesource .= "/$imageinfo->{'package'}" if $imageinfo->{'package'};
  $imagesource .= " [$imageinfo->{'hdrmd5'}]";
  return $imagefile, \%imagebins, $imagesource, \%metas, \%imageorigins;
}

sub getbinaries_buildenv {
  my ($buildinfo, $dir, $srcdir) = @_;

  mkdir_p($dir);
  importbuild() unless defined &Build::queryhdrmd5;

  my @bdeps = grep {$_->{'name'} && $_->{'hdrmd5'}} @{$buildinfo->{'bdep'} || []};
  die("binaries without hdrmd5 in buildenv\n") unless @bdeps == @{$buildinfo->{'bdep'} || []};
  my %needed = map {+"$_->{'name'}.$_->{'hdrmd5'}" => []} @bdeps;
  my %needed_names = map {$_->{'name'} => 1} @bdeps;
  my @needed_names = sort(keys %needed_names);

  my $arch = $buildinfo->{'arch'};
  for my $repo (@{$buildinfo->{'path'} || []}) {
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $projid = $repo->{'project'};
    my $repoid = $repo->{'repository'};
    my $pbvl;
    eval {
      if ($server ne $buildinfo->{'srcserver'}) {
        $pbvl = BSRPC::rpc({
          'uri' => "$server/getpackagebinaryversionlist",
          'timeout' => $gettimeout,
        }, $BSXML::packagebinaryversionlist, "project=$projid", "repository=$repoid", "arch=$arch");
      } else {
        $pbvl = BSRPC::rpc({
          'uri' => "$server/build/$projid/$repoid/$arch",
          'timeout' => $gettimeout,
        }, $BSXML::packagebinaryversionlist, "view=binaryversions");
      }
    };
    undef $pbvl if $@;
    warn($@) if $@;
    next unless $pbvl;
    $pbvl = { map {$_->{'package'} => $_} @{$pbvl->{'binaryversionlist'} || []} };
    for my $packid (sort keys %$pbvl) {
      my $bvl = $pbvl->{$packid};
      for (@{$bvl->{'binary'} || []}) {
        next unless $_->{'name'} && $_->{'hdrmd5'};
        next unless $_->{'name'} =~ /^(?:::import::.*::)?(.*)-[^-]+-[^-]+\.([a-zA-Z][^\.\-]*)\.rpm$/;
	$_->{'repo'} = $repo;
	$_->{'package'} = $packid;
	$_->{'binary'} = $1;
	$_->{'filename'} = $_->{'name'};
	$_->{'name'} = "$1.rpm";
	push @{$needed{"$1.$_->{'hdrmd5'}"}}, $_ if $needed{"$1.$_->{'hdrmd5'}"};
      }
    }
  }
  %needed_names = ();
  $needed_names{$_->{'name'}} = 1 for grep {!@{$needed{"$_->{'name'}.$_->{'hdrmd5'}"}}} @bdeps;
  @needed_names = sort(keys %needed_names);
  if (@needed_names) {
    # maybe we need to look at the full tree as well...
    for my $repo (@{$buildinfo->{'path'} || []}) {
      my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
      my $callers_time = time();
      my @args;
      push @args, "workerid=$workerid" if defined $workerid;
      push @args, "project=$repo->{'project'}";
      push @args, "repository=$repo->{'repository'}";
      push @args, "arch=$arch";
      push @args, 'nometa';
      push @args, map {"module=$_"} @{$buildinfo->{'module'} || []};
      push @args, "binaries=".join(',', @needed_names);
      push @args, "now=$callers_time";
      my $bvl;
      eval {
        $bvl = BSRPC::rpc({
          'uri' => "$server/getbinaryversions",
          'timeout' => $gettimeout,
        }, $BSXML::binaryversionlist, @args);
      };
      undef $bvl if $@;
      warn($@) if $@;
      next unless $bvl;
      for (@{$bvl->{'binary'} || []}) {
        next unless $_->{'hdrmd5'};
        next unless $_->{'name'} =~ /(.*)\.(?:$binsufsre)$/;
	$_->{'repo'} = $repo;
	$_->{'binary'} = $1;
	push @{$needed{"$1.$_->{'hdrmd5'}"}}, $_ if $needed{"$1.$_->{'hdrmd5'}"};
      }
    }
  }
  # ok, all info is collected
  my @missing = grep {!@{$needed{"$_->{'name'}.$_->{'hdrmd5'}"}}} @bdeps;
  die("404 missing packages: ".join(', ', map {+"$_->{'name'}\@$_->{'hdrmd5'}"} @missing)."\n") if @missing;

  my @cacheold;
  my @cachenew;
  my %used;

  # check the cache
  if ($cachedir) {
    for my $needed (sort keys %needed) {
      for my $bv (@{$needed{$needed}}) {
        my $repo = $bv->{'repo'};
        my $projid = $repo->{'project'};
        my $repoid = $repo->{'repository'};
        my ($cacheid, $cachefile) = get_cachefile("$projid/$repoid/$arch", $bv->{'hdrmd5'});
        if (link_or_copy($cachefile, "$dir/$bv->{'name'}")) {
	  my @s = stat("$dir/$bv->{'name'}");
	  die unless @s;
	  my $id = Build::queryhdrmd5("$dir/$bv->{'name'}");
	  if ($id eq $bv->{'hdrmd5'}) {
	    push @cacheold, [$cacheid, $s[7]];
	    die unless $bv->{'name'} =~ /(.*)\.rpm$/;
	    $used{$needed} = { 'name' => $1, 'project' => $projid, 'repository' => $repoid, 'hdrmd5' => $id };
	    delete $needed{$needed};
	    last;
          }
	}
	unlink("$dir/$bv->{'name'}");
      }
    }
  }

  my $downloadsizek = 0;
  for my $needed (keys %needed) {
    $downloadsizek += $needed{$needed}->[0]->{'sizek'} || 0;
  }
  $binariescachehits += @cacheold;
  $binariesdownload += keys %needed;
  $binariesdownloadsize += $downloadsizek;
  if ($cachedir && $downloadsizek * 1024 * 100 > $cachesize) {
    # reserve space
    manage_cache($cachesize - $downloadsizek * 1024);
  }

  # download... slowly...
  for my $needed (sort keys %needed) {
    for my $bv (@{$needed{$needed}}) {
      my $repo = $bv->{'repo'};
      my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
      my $projid = $repo->{'project'};
      my $repoid = $repo->{'repository'};
      my $param;
      my @args;
      unlink("$dir/$bv->{'name'}");
      if ($bv->{'package'}) {
	$param = {
          'uri' => "$server/build/$projid/$repoid/$arch/$bv->{'package'}/$bv->{'filename'}",
          'timeout' => $gettimeout,
	  'receiver' => \&BSHTTP::file_receiver,
	  'filename' => "$dir/$bv->{'name'}",
	};
      } else {
	push @args, map {"module=$_"} @{$buildinfo->{'module'} || []};
	$param = {
          'uri' => "$server/build/$projid/$repoid/$arch/_repository/$bv->{'name'}",
          'timeout' => $gettimeout,
	  'receiver' => \&BSHTTP::file_receiver,
	  'filename' => "$dir/$bv->{'name'}",
	};
      }
      eval {
        BSRPC::rpc($param, undef, @args);
      };
      if ($@) {
	warn($@);
        unlink("$dir/$bv->{'name'}");
	next;
      }
      my @s = stat("$dir/$bv->{'name'}");
      next unless @s;
      my $id = Build::queryhdrmd5("$dir/$bv->{'name'}");
      if ($id ne $bv->{'hdrmd5'}) {
        unlink("$dir/$bv->{'name'}");
	next;
      }
      die unless $bv->{'name'} =~ /(.*)\.rpm$/;
      $used{$needed} = { 'name' => $1, 'project' => $projid, 'repository' => $repoid, 'hdrmd5' => $id };
      my ($cacheid) = get_cachefile("$projid/$repoid/$arch", $bv->{'hdrmd5'});
      push @cachenew, [$cacheid, $s[7], "$dir/$bv->{'name'}"];
      delete $needed{$needed};
      last;
    }
    die("download of $needed failed\n") if $needed{$needed};
  }
  my $outbdep = ($buildinfo->{'outbuildinfo'} || {})->{'bdep'};
  if ($outbdep) {
    for (@bdeps) {
      my $needed = "$_->{'name'}.$_->{'hdrmd5'}";
      my $obdep = $used{$needed};
      die("did not use package $needed?\n") unless $obdep;
      push @$outbdep, $obdep;
    }
  }

  manage_cache($cachesize, \@cacheold, \@cachenew) if $cachedir;
  return ();	# no meta
}

sub getimagemode {
  my ($buildinfo) = @_;
  my $file = $buildinfo->{'file'};
  $file =~ s/^_service:.*://;
  my $imagemode;
  $imagemode = 'kiwiimage' if $file =~ /\.kiwi$/;
  $imagemode = 'kiwiproduct' if $imagemode && $buildinfo->{'imagetype'} && $buildinfo->{'imagetype'}->[0] eq 'product';
  $imagemode = 'docker' if $file =~ /^Dockerfile(?:\.|$)/;
  $imagemode = 'fissile' if $file eq 'fissile.yml';
  $imagemode = 'productcompose' if $file =~ /\.productcompose$/;
  return $imagemode;
}

sub getbinaries_containers {
  my ($buildinfo, $dir, $srcdir, $origins) = @_;

  my $basecontainerannotationxml;
  my @meta;
  for my $bdep (grep {$_->{'name'} =~ /^container:/} @{$buildinfo->{'bdep'} || []}) {
    my $repo;
    if ($bdep->{'project'} && $bdep->{'repository'}) {
      $repo = (grep {$_->{'project'} eq $bdep->{'project'} && $_->{'repository'} eq $bdep->{'repository'}} @{$buildinfo->{'containerpath'} || []})[0];
      die("getbinaries: job has no container repository matching $bdep->{'project'}/$bdep->{'repository'}\n") unless $repo;
    } else {
      $repo = ($buildinfo->{'containerpath'} || [])->[0];
      die("getbinaries: job has no container repository\n") unless $repo;
    }
    my $ddir = "$srcdir/containers/$repo->{'project'}/$repo->{'repository'}";
    mkdir_p($ddir);
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $callers_time = time();
    my @args;
    push @args, "workerid=$workerid" if defined $workerid;
    push @args, "project=$repo->{'project'}";
    push @args, "repository=$repo->{'repository'}";
    push @args, "arch=$buildinfo->{'arch'}";
    push @args, "withannotation=1" if $server ne $buildinfo->{'srcserver'};
    push @args, "now=$callers_time";
    my $res = BSRPC::rpc({
      'uri' => "$server/getbinaries",
      'directory' => $ddir,
      'withmd5' => 1,
      'timeout' => $gettimeout,
      'receiver' => \&BSHTTP::cpio_receiver,
    }, undef, @args, "binaries=$bdep->{'name'}");
    die("Error\n") unless ref($res);
    
    my $annotationfile = (map {$_->{'name'}} grep {$_->{'name'} =~ /\.annotation$/} @$res)[0];
    $res = [ grep {$_->{'name'} ne '.errors' && $_->{'name'} !~ /\.annotation$/} @$res ];
    die("404 getbinaries: missing packages: $bdep->{'name'}\n") unless @$res == 1;
    my $tarname = $res->[0]->{'name'};
    my $tarmd5 = $res->[0]->{'md5'};
    add_slsa_materials_binary($buildinfo, "$repo->{'project'}/$repo->{'repository'}/$buildinfo->{'arch'}", $tarname, "$ddir/$tarname", 'containers') if $buildinfo->{'slsaprovenance'};
    my $imagemode = getimagemode($buildinfo);
    if ($imagemode eq 'kiwiimage' && $tarname =~ /\.tar$/) {
      system('gzip', '-f', '-1', "$ddir/$tarname") && die("kiwi container compression: $?\n");
      $tarname .= '.gz';
      die("kiwi container compression did not work\n") unless -f "$ddir/$tarname";
    }
    push @{$buildinfo->{'containerurl'}}, "dir://./containers/$repo->{'project'}/$repo->{'repository'}/$tarname";
    push @meta, "$tarmd5  $repo->{'project'}/$repo->{'repository'}/$bdep->{'name'}";
    my $bin = {
      'name' => $bdep->{'name'},
      'project' => $repo->{'project'},
      'repository' => $repo->{'repository'},
      'arch' => $buildinfo->{'arch'},
    };
    undef $basecontainerannotationxml;
    if ($annotationfile) {
      my $annotationxml = BSUtil::readstr("$ddir/$annotationfile", 1);
      my $annotation = $annotationxml ? BSUtil::fromxml($annotationxml, $BSXML::binannotation, 1) : undef;
      if ($annotation) {
	$basecontainerannotationxml = $annotationxml;	# this relies on the base container coming last
	die("annotation does not match container\n") if $annotation->{'hdrmd5'} && $annotation->{'hdrmd5'} ne $tarmd5;
	for (qw{package version release binaryarch disturl buildhost buildtime binaryid}) {
	  $bin->{$_} = $annotation->{$_} if defined($annotation->{$_}) && $annotation->{$_} ne '';
	}
      }
    }
    $bin->{'binaryarch'} ||= $bin->{'arch'} || 'noarch';
    push @{$origins->{':basecontainers'}}, $bin;
  }

  $basecontainerannotationxml ||= $buildinfo->{'containerannotation'};
  if ($basecontainerannotationxml) {
    mkdir_p("$srcdir/containers");
    writestr("$srcdir/containers/annotation", undef, $basecontainerannotationxml);
  }

  return @meta;
}

# TODO: add provenance materials
# Use for kiwi appliances, docker or fissile builds
sub getbinaries_image {
  my ($buildinfo, $dir, $srcdir, $origins) = @_;

  mkdir_p($dir);

  # we need the Build package for queryhdrmd5
  importbuild() unless defined &Build::queryhdrmd5;

  my @bdep = grep {($_->{'repoarch'} || '') ne 'src'} @{$buildinfo->{'bdep'} || []};
  my %bdep_notmeta = map {$_->{'name'} => 1} grep {$_->{'notmeta'}} @bdep;
  my %bdep_noinstall = map {$_->{'name'} => 1} grep {$_->{'noinstall'}} @bdep;
  @bdep = map {$_->{'name'}} @bdep;

  my @todo = @bdep;
  die("no binaries needed for this package?\n") unless @todo;
  my @meta;
  my $modules = $buildinfo->{'module'};

  # fetch environment in two path mode
  if ($buildinfo->{'syspath'}) {
    my $syspatharch = $buildinfo->{'crossarch'} || $buildinfo->{'arch'};
    @todo = grep {!$bdep_noinstall{$_}} @bdep;
    for my $repo (@{$buildinfo->{'syspath'} || []}) {
      last unless @todo;
      my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
      my $got = getbinaries_cache($dir, $server, $repo->{'project'}, $repo->{'repository'}, $syspatharch, 1, \@todo, $modules);
      if ($buildinfo->{'slsaprovenance'}) {
	for my $n (sort keys %$got) {
	  my $f = $got->{$n};
	  add_slsa_materials_binary($buildinfo, "$repo->{'project'}/$repo->{'repository'}/$syspatharch", $f->{'name'}, "$dir/$f->{'name'}") if $buildinfo->{'slsaprovenance'};
	}
      }
      @todo = grep {!$got->{$_}} @todo;
    }
    die("404 getbinaries: missing packages: @todo\n") if @todo;
    @todo = grep {$bdep_noinstall{$_} || !$bdep_notmeta{$_}} @bdep;
  }

  # we fetch the containes in a second step
  @todo = grep {!/^container:/} @todo;

  # unordered repositories feature
  my %todo_origins;
  for (@{$buildinfo->{'bdep'} || []}) {
    next unless $_->{'project'} && $_->{'repository'};
    next if $_->{'repoarch'} || '' eq 'src';
    next if $_->{'name'} =~ /^container:/;
    next if !$_->{'noinstall'} && $_->{'notmeta'};
    $todo_origins{$_->{'name'}} = "$_->{'project'}/$_->{'repository'}";
  }

  my %bvls;		# bvl cache, the query is not free

  for my $repo (@{$buildinfo->{'path'} || []}) {
    my $repoprp = "$repo->{'project'}/$repo->{'repository'}";
    my $ddir = "$srcdir/repos/$repoprp";
    mkdir_p($ddir);
    my @todo_repo = @todo;
    if (%todo_origins) {
      @todo_repo = grep {!$todo_origins{$_} || $todo_origins{$_} eq $repoprp} @todo_repo;
    }
    next if !@todo_repo;
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $arch = $buildinfo->{'arch'};
    my $got = getbinaries_cache($ddir, $server, $repo->{'project'}, $repo->{'repository'}, $arch, 1, \@todo_repo, $modules, $bvls{"$repoprp/$arch"});
    for my $n (sort keys %$got) {
      my $f = $got->{$n};
      if (!$bdep_notmeta{$n}) {
	my $id = Build::queryhdrmd5("$ddir/$f->{'name'}") || "deaddeaddeaddeaddeaddeaddeaddead";
	push @meta, "$id  $repoprp/$n";
	# sigh, no package information available yet...
	$origins->{$n} = "$repoprp/$buildinfo->{'arch'}" if $origins;
      }
      if (!$buildinfo->{'syspath'} && !$bdep_noinstall{$n}) {
	if (!-e "$dir/$f->{'name'}") {
	  link_or_copy("$ddir/$f->{'name'}", "$dir/$f->{'name'}") || die("link_or_copy $ddir/$f->{'name'} $dir/$f->{'name'}: $!\n");
	  add_slsa_materials_binary($buildinfo, "$repo->{'project'}/$repo->{'repository'}/$arch", $f->{'name'}, "$dir/$f->{'name'}") if $buildinfo->{'slsaprovenance'};
	}
	unlink("$ddir/$f->{'name'}") if $bdep_notmeta{$n};
      }
    }
    @todo = grep {!$got->{$_}} @todo;
  }
  die("404 getbinaries: missing packages: @todo\n") if @todo;

  # fetch the base containers
  push @meta, getbinaries_containers($buildinfo, $dir, $srcdir, $origins);

  # sort the meta
  @meta = sort {substr($a, 34) cmp substr($b, 34) || $a cmp $b} @meta;
  return @meta;
}

sub getbinaries {
  my ($buildinfo, $dir, $srcdir, $preinstallimagedata, $origins) = @_;

  return getbinaries_buildenv($buildinfo, $dir, $srcdir) if $buildinfo->{'hasbuildenv'};

  my $imagemode = getimagemode($buildinfo);
  return getbinaries_image($buildinfo, $dir, $srcdir, $origins) if $imagemode;

  mkdir_p($dir);

  # we need the Build package for queryhdrmd5
  importbuild() unless defined &Build::queryhdrmd5;

  my @bdep = grep {($_->{'repoarch'} || '') ne 'src'} @{$buildinfo->{'bdep'} || []};
  my @todo = map {$_->{'name'}} @bdep;

  # split into todo/todo_sysroot in cross mode
  my $cross = $buildinfo->{'crossarch'};
  my @todo_sysroot = map {$_->{'name'}} grep {$_->{'sysroot'}} @bdep;
  die("sysroot packages need a crossarch\n") if @todo_sysroot && !$cross;
  @todo = map {$_->{'name'}} grep {!$_->{'sysroot'}} @bdep if @todo_sysroot;
  die("no binaries needed for this package?\n") unless @todo;

  my $outbdep = ($buildinfo->{'outbuildinfo'} || {})->{'bdep'};
  my %done;

  my %meta;
  my $packid = $buildinfo->{'package'};
  my $projid = $buildinfo->{'project'};
  my $repoid = $buildinfo->{'repository'};
  my $modules = $buildinfo->{'module'};
  my %bvls;		# bvl cache, the query is not free

  # get a preinstall image if available
  my ($imagefile, $imagebins);
  if (@todo && $preinstallimagedata && $Build::Features::preinstallimage) {
    my ($imagesource, $imagemetas, $imageorigins);
    ($imagefile, $imagebins, $imagesource, $imagemetas, $imageorigins) = getpreinstallimage($buildinfo, $dir, \@todo, \%bvls);
    if ($imagefile && $imagebins && $imagemetas) {
      $preinstallimagedata->{'imagename'} = $imagefile;
      $preinstallimagedata->{'imagebins'} = $imagebins;
      $preinstallimagedata->{'imagesource'} = $imagesource;
      $meta{$_} = 1 for keys %$imagemetas;
      if ($outbdep) {
	# record preinstallimage data
        for (grep {$imagebins->{$_}} @todo) {
	  my $bv = $imagebins->{$_};
	  my $obdep = { 'name' => $_, 'hdrmd5' => $bv->{'hdrmd5'}, 'project' => $imageorigins->{$_}->[0], 'repository' => $imageorigins->{$_}->[1] };
	  if ($bv->{'evr'}) {
	    $obdep->{'version'} = $bv->{'evr'};
	    $obdep->{'epoch'} = $1 if $obdep->{'version'} =~ s/^([0-9]+)://;
	    $obdep->{'release'} = $1 if $obdep->{'version'} =~ s/-([^\-]+)$//;
	    $obdep->{'arch'} = $bv->{'arch'} if $bv->{'arch'};
	  }
	  push @$outbdep, $obdep;
        }
      }
      # remove image packages from todo
      @todo = grep {!$imagebins->{$_}} @todo;
    }
  }
  $imagebins ||= {};

  # get all the (target) packages
  @todo = @todo_sysroot if $cross;
  for my $repo (@{$buildinfo->{'path'} || []}) {
    last unless @todo;
    my $repoprp = "$repo->{'project'}/$repo->{'repository'}";
    my $nometa = $repo->{'project'} ne $projid || $repo->{'repository'} ne $repoid ? 1 : 0;
    $nometa = 1 if $buildinfo->{'forcebinaryidmeta'};
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $arch = $buildinfo->{'arch'};
    my $binaryprefix = $cross ? '__sysroot__' : '';
    my $got = getbinaries_cache($dir, $server, $repo->{'project'}, $repo->{'repository'}, $arch, $nometa, \@todo, $modules, $bvls{"$repoprp/$arch"}, $binaryprefix);
    for (sort keys %$got) {
      my $gotpkg = $got->{$_};
      $done{$_} = $gotpkg->{'name'};
      $meta{$_} = 1 if !$nometa && $gotpkg->{'meta'};
      if ($outbdep) {
	my $obdep = { 'name' => $_, 'project' => $repo->{'project'}, 'repository' => $repo->{'repository'} };
	if ($cross) {
	  substr($obdep->{'name'}, 0, length($binaryprefix), '');
	  $obdep->{'sysroot'} = 1;
	}
	$obdep->{'hdrmd5'} = $gotpkg->{'hdrmd5'} if $gotpkg->{'hdrmd5'};
	if ($gotpkg->{'evr'}) {
	  $obdep->{'version'} = $gotpkg->{'evr'};
	  $obdep->{'epoch'} = $1 if $obdep->{'version'} =~ s/^([0-9]+)://;
	  $obdep->{'release'} = $1 if $obdep->{'version'} =~ s/-([^\-]+)$//;
	  $obdep->{'arch'} = $gotpkg->{'arch'} if $gotpkg->{'arch'};
	}
	push @$outbdep, $obdep;
      }
      if (!$cross && $origins) {
	# sigh, no package information available yet...
	$origins->{$_} = "$repoprp/$arch";
      }
      if ($buildinfo->{'slsaprovenance'}) {
        if ($cross) {
          add_slsa_materials_binary($buildinfo, "$repo->{'project'}/$repo->{'repository'}/$arch", substr($gotpkg->{'name'}, length($binaryprefix)), "$dir/$gotpkg->{'name'}", 'sysroot');
        } else {
          add_slsa_materials_binary($buildinfo, "$repo->{'project'}/$repo->{'repository'}/$arch", $gotpkg->{'name'}, "$dir/$gotpkg->{'name'}");
        }
      }
    }
    @todo = grep {!$done{"$binaryprefix$_"}} @todo;
  }
  die("404 getbinaries: missing packages: @todo\n") if @todo;

  # now get the native packages in cross mode
  @todo = map {$_->{'name'}} grep {!$_->{'sysroot'}} @bdep if $cross;
  @todo = grep {!$imagebins->{$_}} @todo;
  for my $repo (@{$buildinfo->{'syspath'} || []}) {
    last unless @todo;
    my $repoprp = "$repo->{'project'}/$repo->{'repository'}";
    my $server = $repo->{'server'} || $buildinfo->{'reposerver'};
    my $arch = $cross;
    my $got = getbinaries_cache($dir, $server, $repo->{'project'}, $repo->{'repository'}, $arch, 1, \@todo, $modules, $bvls{"$repoprp/$arch"});
    for (sort keys %$got) {
      my $gotpkg = $got->{$_};
      $done{$_} = $gotpkg->{'name'};
      if ($outbdep) {
	my $obdep = { 'name' => $_, 'project' => $repo->{'project'}, 'repository' => $repo->{'repository'} };
	$obdep->{'hdrmd5'} = $gotpkg->{'hdrmd5'} if $gotpkg->{'hdrmd5'};
	if ($gotpkg->{'evr'}) {
	  $obdep->{'version'} = $gotpkg->{'evr'};
	  $obdep->{'epoch'} = $1 if $obdep->{'version'} =~ s/^([0-9]+)://;
	  $obdep->{'release'} = $1 if $obdep->{'version'} =~ s/-([^\-]+)$//;
	  $obdep->{'arch'} = $gotpkg->{'arch'} if $gotpkg->{'arch'};
	}
	push @$outbdep, $obdep;
      }
      if ($origins) {
        # sigh, no package information available yet...
        $origins->{$_} = "$repoprp/$arch";
      }
      add_slsa_materials_binary($buildinfo, "$repo->{'project'}/$repo->{'repository'}/$arch", $gotpkg->{'name'}, "$dir/$gotpkg->{'name'}") if $buildinfo->{'slsaprovenance'};
    }
    @todo = grep {!$done{$_}} @todo;
  }
  die("404 getbinaries: missing native packages: @todo\n") if @todo;

  # everything is downloaded, now generate meta data
  my @meta;

  # add meta for all native deps in cross mode
  if ($cross) {
    for my $dep (map {$_->{'name'}} grep {!$_->{'notmeta'} && !$_->{'sysroot'}} @bdep) {
      my $id = ($imagebins->{$dep} || {})->{'hdrmd5'} || Build::queryhdrmd5("$dir/$done{$dep}") || "deaddeaddeaddeaddeaddeaddeaddead";
      push @meta, "$id  $cross:$dep";
    }
  }

  # we carefully prune entries here to keep the memory usage down
  # so that coolo's image builds work
  # - do not prune entries that have one of our subpacks in the
  #   path as they are handled in a special way
  # - use the same order as the code in BSBuild
  my %mseen;
  my @subp = @{$buildinfo->{'subpack'} || []};
  my $subpackre = '';
  for (@subp) {
    $subpackre .= "|/\Q$_\E/";
  }
  if ($subpackre) {
    $subpackre = substr($subpackre, 1);
    $subpackre = qr/$subpackre/;
  }
  my @metadeps;
  if ($cross) {
    @metadeps = map {$_->{'name'}} grep {!$_->{'notmeta'} && $_->{'sysroot'}} @bdep;
  } else {
    @metadeps = map {$_->{'name'}} grep {!$_->{'notmeta'}} @bdep;
  }
  for my $dep (sort {"$a/" cmp "$b/"} @metadeps) {
    my $prefixeddep = $cross ? "__sysroot__$dep" : $dep;
    my $m;
    $m = readstr("$dir/$prefixeddep.meta", 1) if $meta{$prefixeddep};
    if (!$m) {
      my $id = ($imagebins->{$dep} || {})->{'hdrmd5'} || Build::queryhdrmd5("$dir/$done{$prefixeddep}") || "deaddeaddeaddeaddeaddeaddeaddead";
      push @meta, "$id  $dep";
    } else {
      chomp $m;
      my @m = split("\n", $m);
      # do not include our own build results
      next if $m[0] =~ /  \Q$packid\E$/s;
      $m[0] =~ s/  .*/  $dep/;
      push @meta, shift @m;
      if ($subpackre && "/$dep/" =~ /$subpackre/) {
	s/  /  $dep\// for @m;
	push @meta, @m;
      } else {
	for (@m) {
	  next if $mseen{$_};
	  my $x = $_;
	  s/  /  $dep\//;
	  $mseen{$x} = 1 unless $subpackre && "$_/" =~ /$subpackre/;
	  push @meta, $_;
	}
      }
    }
  }
  push @meta, $buildinfo->{'modularity_meta'} if $buildinfo->{'modularity_meta'};
  BSBuild::setgenmetaalgo($buildinfo->{'genmetaalgo'}) if $buildinfo->{'genmetaalgo'};
  @meta = BSBuild::gen_meta($buildinfo->{'subpack'} || [], @meta);
  return @meta;
}

sub getlinkedccache {
  my ($buildinfo, $odir) = @_;

  mkdir_p($odir) || die("mkdir_p $odir: $!\n");
  for my $other (@{$buildinfo->{'path'}}) {
    if ($other->{'server'} ne $buildinfo->{'srcserver'}) {
      my $oprojid = $other->{'project'};
      my $orepoid = $other->{'repository'};
      my $opackid = $buildinfo->{'package'};

      next if $oprojid eq $buildinfo->{'project'} && $orepoid eq $buildinfo->{'repository'};

      # do extra work for maintenance incidents
      if ($buildinfo->{'releasename'}) {
        $opackid = $buildinfo->{'releasename'};
        eval {
          my $link = BSRPC::rpc("$buildinfo->{'srcserver'}/source/$oprojid/$opackid/_link", $BSXML::link, 'noajax=1');
          $opackid = $link->{'package'} if $link->{'package'} && !$link->{'project'};
        };
        warn($@) if $@ && $@ !~ /^404/;
      }

      my $res;
      eval {
        $res = BSRPC::rpc({
          'uri' => "$other->{'server'}/build/$oprojid/$orepoid/$buildinfo->{'arch'}/$opackid/_ccache.tar",
          'filename' => "$odir/_ccache.tar",
          'timeout' => $gettimeout,
          'receiver' => \&BSHTTP::file_receiver,
        }, undef);
      };
      if ($@ && $@ !~ /^404/) {
        warn($@);
      } else {
        $binariesdownload += 1;
        $binariesdownloadsize += int($res->{'size'} / 1024);
        last;
      }
    }
  }
  rmdir($odir);	# if not needed
}

sub getoldpackages {
  my ($buildinfo, $odir, $useccache) = @_;

  # get old package build for compare and diffing tools
  mkdir_p($odir) || die("mkdir_p $odir: $!\n");
  my $res;
  eval {
    $res = BSRPC::rpc({
      'uri' => "$buildinfo->{'reposerver'}/build/$buildinfo->{'project'}/$buildinfo->{'repository'}/$buildinfo->{'arch'}/$buildinfo->{'package'}",
      'directory' => $odir,
      'timeout' => $gettimeout,
      'receiver' => \&BSHTTP::cpio_receiver,
    }, undef, 'view=cpio', 'noajax=1', 'noimport=1', "withccache=$useccache");
  };
  die("getoldpackages: $@") if $@;
  rmdir($odir);	# if not needed
  for my $file (@$res) {
    $binariesdownload += 1;
    $binariesdownloadsize += int($file->{'size'} / 1024);
  }
}

sub readbuildenv {
  my ($buildinfo, $src) = @_;
  my $bi = readxml("$src/_buildenv.$buildinfo->{'repository'}.$buildinfo->{'arch'}", $BSXML::buildinfo, 1);
  $bi ||= readxml("$src/_buildenv", $BSXML::buildinfo, 1) unless $buildinfo->{'followupfile'};
  return unless $bi;
  die("buildenv does not work for kiwi builds\n") if $buildinfo->{'file'} =~ /\.kiwi$/;
  # have a buildenv! patch stuff!
  $buildinfo->{'hasbuildenv'} = 1;
  # work around buildenv generation bug
  $_->{'name'} =~ s/\.rpm$// for @{$bi->{'bdep'} || []};
  if (($buildinfo->{'signflavor'} || '') eq '_buildenv') {
    die("Need to have an outbuildinfo\n") unless $buildinfo->{'outbuildinfo'};
    delete $buildinfo->{'signflavor'};
    delete $buildinfo->{'outbuildinfo'}->{'signflavor'};
    $buildinfo->{'signflavor'} = $bi->{'signflavor'} if defined $bi->{'signflavor'};
    $buildinfo->{'outbuildinfo'}->{'signflavor'} = $bi->{'signflavor'} if defined $bi->{'signflavor'};
  }
  for (qw{error versrel bcnt release bdep config}) {
    $buildinfo->{$_} = $bi->{$_} if defined $bi->{$_};
  }
  for (qw{versrel bcnt release}) {
    $buildinfo->{'outbuildinfo'}->{$_} = $bi->{$_} if $buildinfo->{'outbuildinfo'} && defined $bi->{$_};
  }
  if (!grep {$_->{'preinstall'}} @{$buildinfo->{'bdep'} || []}) {
    # fixup preinstall/vminstall...
    importbuild() unless defined &Build::queryhdrmd5;
    my $server = $buildinfo->{'srcserver'} || $srcserver;
    my $bconf;
    if (defined $buildinfo->{'config'}) {
      $bconf = $buildinfo->{'config'};
    } else {
      $bconf = BSRPC::rpc("$server/getconfig", undef, "project=$buildinfo->{'project'}", "repository=$buildinfo->{'repository'}");
    }
    $bconf = Build::read_config($buildinfo->{'arch'}, [ split("\n", $bconf) ]);
    my %pdeps = map {$_ => 1} Build::get_preinstalls($bconf);
    my %vmdeps = map {$_ => 1} Build::get_vminstalls($bconf);
    my %runscripts = map {$_ => 1} Build::get_runscripts($bconf);
    for (@{$buildinfo->{'bdep'} || []}) {
      $_->{'preinstall'} = 1 if $pdeps{$_->{'name'}};
      $_->{'vminstall'} = 1 if $vmdeps{$_->{'name'}};
      $_->{'runscripts'} = 1 if $runscripts{$_->{'name'}};
    }
  }
}

sub kiwiproduct_patch_recipe  {
  my ($buildinfo, $kiwifile, $meta) = @_;

  require Build::SimpleXML;
  my $kiwi_xml = readstr($kiwifile);
  my $kiwi = Build::SimpleXML::parse($kiwi_xml, 'record' => 1);
  my @patchlist;
  die("no image elsement in kiwi file\n") unless $kiwi->{'image'};
  my $image = $kiwi->{'image'}->[0];
  die("no instsource section in kiwi file\n") unless $image->{'instsource'};
  my $instsource = $image->{'instsource'}->[0];

  # patch "MEDIUM_NAME" product variable
  my $productoptions = $instsource->{'productoptions'} ? $instsource->{'productoptions'}->[0] : {};
  for my $productvar (@{$productoptions->{'productvar'} || []}) {
    if ($productvar->{'name'} eq 'MEDIUM_NAME') {
      my $mediumbase = $productvar->{'_content'};
      my $cicnt = $buildinfo->{'versrel'};
      $cicnt =~ s/.*-//;
      if ($cicnt eq '1') {
	# simple case for standard productconverter case
	$mediumbase .= sprintf("-Build%04d", $buildinfo->{'bcnt'} || 0);
      } else {
	# standard CI_CNT-BCNT writing
	$mediumbase .= sprintf("-Build%s.%d", $cicnt, $buildinfo->{'bcnt'} || 0);
      }    
      my $newxml = Build::SimpleXML::unparse({ 'productvar' => [ { 'name' => 'BUILD_ID', '_content' => $mediumbase }, { 'name' => 'MEDIUM_NAME', '_content' => "$mediumbase-Media" } ] }, 'indent' => '      ');
      $newxml =~ s/^\s+//;
      $newxml =~ s/\n$//s;
      push @patchlist, [ $productvar->{'_start'}, $productvar->{'_end'}, $newxml ];
    }
  }

  # expand obsrepositories:/ in the instrepo path
  for my $repo (@{$instsource->{'instrepo'} || []}) {
    next unless $repo->{'source'} && $repo->{'source'}->[0]->{'path'} eq 'obsrepositories:/';
    my @rp;
    my $prio = 0;
    for my $path (@{$buildinfo->{'path'} || []}) {
      $prio = $prio + 1;
      my $h = { 'source' => [ { 'path' => "obs://$path->{'project'}/$path->{'repository'}" } ] };
      $h->{'priority'} = $prio;
      $h->{'name'} = "obsrepositories_$prio";
      push @rp, $h;
    }
    my $newxml = Build::SimpleXML::unparse({ 'instrepo' => \@rp }, 'indent' => '    ');
    $newxml =~ s/^\s+//;
    $newxml =~ s/\n$//s;
    push @patchlist, [ $repo->{'_start'}, $repo->{'_end'}, $newxml ];
  }

  # expand a "take all packages" for products
  for my $repopackages (@{$instsource->{'repopackages'} || []}) {
    for my $repopackage (@{$repopackages->{'repopackage'} || []}) {
      next unless $repopackage && $repopackage->{'name'} eq '*';
      my @rqarch;
      if ($instsource->{'architectures'}) {
	for (@{$instsource->{'architectures'}->[0]->{'requiredarch'} || []}) {
	  push @rqarch, $_->{'ref'} if $_->{'ref'};
	}
      }
      my %allpkgs;
      for my $m (@$meta) {
	# md5  proj/rep/arch/pack/bin.arch
	my @s = split('/', $m);
	next unless $s[-1] =~ /^(.*)\.([^\.]*)$/;
	$allpkgs{$1}->{$2} = 1 unless $2 eq 'src' || $2 eq 'nosrc';
      }
      my @rp;
      for my $pkg (sort keys %allpkgs) {
	# exclude blind take of all debug packages. They will be taken
	# automatically if a configured debug medium exists.
	next if $pkg =~ /-debug(?:info|source)(?:-32bit|-64bit|-x86)?$/;
	my @a = map { $_ eq 'noarch' ? @rqarch : ($_) } sort keys %{$allpkgs{$pkg}};
	push @rp, { 'name' => $pkg, 'arch' => join(',', sort(BSUtil::unify(@a))), '_order' => [ 'name', 'arch' ] };
      }
      my $newxml = Build::SimpleXML::unparse({ 'repopackage' => \@rp }, 'indent' => '      ');
      $newxml =~ s/^\s+//;
      $newxml =~ s/\n$//s;
      push @patchlist, [ $repopackage->{'_start'}, $repopackage->{'_end'}, $newxml ];
    }
  }

  my $patchoff = 0;
  for my $patch (sort {$a->[0] <=> $b->[0]} @patchlist) {
    substr($kiwi_xml, $patch->[0] + $patchoff, $patch->[1] - $patch->[0], $patch->[2]);
    $patchoff += length($patch->[2]) - ($patch->[1] - $patch->[0]);
  }

  writestr($kiwifile, undef, $kiwi_xml);
}

sub kiwiproduct_is_rpmhdrsonly {
  my ($kiwifile) = @_;

  require Build::SimpleXML;
  my $kiwi_xml = readstr($kiwifile);
  my $kiwi = Build::SimpleXML::parse($kiwi_xml);
  die("no image elsement in kiwi file\n") unless $kiwi->{'image'};
  my $image = $kiwi->{'image'}->[0];
  die("no instsource section in kiwi file\n") unless $image->{'instsource'};
  my $instsource = $image->{'instsource'}->[0];
  my $productoptions = $instsource->{'productoptions'} ? $instsource->{'productoptions'}->[0] : {};
  for my $productvar (@{$productoptions->{'productvar'} || []}) {
    return 1 if $productvar->{'name'} eq 'RPMHDRS_ONLY' and $productvar->{'_content'} eq 'true';
  }
  return 0;
}

sub xmlescape {
  my ($d) = @_;
  $d =~ s/&/&amp;/sg;
  $d =~ s/</&lt;/sg;
  $d =~ s/>/&gt;/sg;
  $d =~ s/"/&quot;/sg;
  return $d;
}

sub addreportdata {
  my ($report, $buildinfo) = @_;
  if ($buildinfo->{'versrel'}) {
    $report->{'version'} = $buildinfo->{'versrel'};
    $report->{'version'} =~ s/-[^-]*$//s;
  }
  $report->{'release'} = $buildinfo->{'release'} if defined $buildinfo->{'release'};
  $report->{'buildtime'} = $buildinfo->{'buildtime'} if $buildinfo->{'buildtime'};
  $report->{'buildhost'} = $buildinfo->{'buildhost'} if $buildinfo->{'buildhost'};
  $report->{'disturl'} = $buildinfo->{'disturl'} if $buildinfo->{'disturl'};
}

sub cpeid_from_rpm {
  my ($rpm_file) = @_;

  my $header = Build::query($rpm_file, "evra" => 1);
  my ($cpeid) = grep {/^product-cpeid\(\)/} @{$header->{'provides'} || []};
  $cpeid =~ s/.* = // if $cpeid;
  return $cpeid ? BSHTTP::urldecode($cpeid) : undef;
}

# parse all .report files and create a channel file from them
# also update the data in the .report files with the binary origins
sub createchannel {
  my ($buildinfo, $dir, $kiwiorigins, $srcdir) = @_;
  my @reports = grep {/\.report$/} ls($dir);
  my %channel;
  my $hasunknown;
  my $hasconflict = '';
  my %channelnames;
  for my $reportfile (@reports) {
    my $report = readxml("$dir/$reportfile", $BSXML::report, 1);
    next unless $report;
    addreportdata($report, $buildinfo);
    for my $bin (@{$report->{'binary'} || []}) {
      # update missing entries
      my $prpap = $kiwiorigins->{$bin->{'_content'}};
      if (!$prpap) {
	$prpap = 'UNKNOWN/UNKNOWN/UNKNOWN/UNKNOWN';
	$hasunknown = 1;
      } elsif (!defined($bin->{'cpeid'}) && $bin->{'name'} =~ /-release$/) {
        # current product builders should provide cpeid already
        # add them for old tools, but only for -release packages for speed reasons
        my $rpm_file = $bin->{'_content'};
        my $buildsrcdir = "$buildroot/.build.packages/SOURCES";
        $buildsrcdir = "$buildroot/.build-srcdir" if $vm =~ /(xen|kvm|qemu|emulator)/;
        $rpm_file =~ s/obs:\//$buildsrcdir\/repos/;
        if (-e $rpm_file) {
          my $cpeid = cpeid_from_rpm($rpm_file);
          $bin->{'cpeid'} = $cpeid if $cpeid;
        }
      }

      my ($projid, $repoid, $arch, $packid) = split('/', $prpap, 4);
      $bin->{'project'} = $projid;
      $bin->{'repository'} = $repoid;
      $bin->{'arch'} = $arch;
      $bin->{'package'} = $packid;
      # do not put src rpms into channel
      next if $bin->{'binaryarch'} eq 'src' || $bin->{'binaryarch'} eq 'nosrc';
      if ($channelnames{$bin->{'name'}} && $channelnames{$bin->{'name'}} ne "$projid/$repoid/$arch/$packid") {
	$hasconflict .= "$bin->{'name'}: $channelnames{$bin->{'name'}} - $projid/$repoid/$arch/$packid\n";
      }
      $channelnames{$bin->{'name'}} = "$projid/$repoid/$arch/$packid";
      $channel{"$projid/$repoid/$arch"}->{"$packid/$bin->{'name'}"} = $bin;
    }
    writexml("$dir/.$reportfile", "$dir/$reportfile", $report, $BSXML::report);
  }
  # sort helper
  my @chansplits;
  for my $prpa (keys %channel) {
    my ($projid, $repoid, $arch) = split('/', $prpa, 3);
    push @chansplits, {
      'project' => $projid,
      'repository' => $repoid,
      'arch' => $arch,
      'prpa' => $prpa,
    };
  }
  @chansplits = sort {
    $a->{'project'} cmp $b->{'project'} ||
    $a->{'repository'} cmp $b->{'repository'} ||
    $a->{'arch'} cmp $b->{'arch'}
  } @chansplits;

  # write the channel
  # we directly write it so we can format a bit
  my $c = '';
  $c .= "\n<!-- WARNING: this file contains UNKNOWN entries -->\n\n" if $hasunknown;
  $c .= "\n<!-- WARNING: this file contains the following conflicts:\n\n$hasconflict\n-->\n\n" if $hasconflict;
  $c .= "<channel>\n";
  for my $chansplit (@chansplits) {
    $c .= "  <binaries";
    for (qw{project repository package arch name supportstatus}) {
      $c .= " $_=\"".xmlescape($chansplit->{$_}).'"' if defined $chansplit->{$_};
    }
    $c .= ">\n";
    my @bins = values(%{$channel{$chansplit->{'prpa'}}});
    for my $bin (sort {
			$a->{'project'} cmp $b->{'project'} ||
			$a->{'repository'} cmp $b->{'repository'} ||
			$a->{'arch'} cmp $b->{'arch'} ||
			$a->{'package'} cmp $b->{'package'} ||
			$a->{'name'} cmp $b->{'name'} ||
			$a->{'_content'} cmp $b->{'_content'}
                      } @bins) {
      $c .= "    <binary";
      my $x = length("    <binary") + 1;
      my $first = 0;
      for (qw{project repository package arch name supportstatus}) {
	next unless defined($bin->{$_});
	next if defined($chansplit->{$_}) && $chansplit->{$_} eq $bin->{$_};
	if ($first == 1 && $x < 48) {
          my $pad = ' ' x (48 - $x);
          $c .= $pad;
          $x += length($pad);
        }
        if ($first == 2 && $x < 88) {
          my $pad = ' ' x (88 - $x);
          $c .= $pad;
          $x += length($pad);
        }
	my $str = " $_=\"".xmlescape($bin->{$_}).'"';
        $c .= $str;
        $x += length($str);
        $first++;
      }   
      $c .= "/>\n";
    }   
    $c .= "  </binaries>\n\n";
  }
  $c .= "</channel>\n";
  writestr("$dir/._channel", "$dir/_channel", $c);
}

# add origin information to the report
sub annotatereport {
  my ($buildinfo, $dir, $kiwiorigins) = @_;
  my @reports = grep {/\.report$/} ls($dir);
  for my $reportfile (@reports) {
    my $report = readxml("$dir/$reportfile", $BSXML::report, 1);
    next unless $report;
    addreportdata($report, $buildinfo);
    for my $bin (@{$report->{'binary'} || []}) {
      # update entries
      my $prpap = $kiwiorigins->{$bin->{'_content'}};
      $prpap ||= 'UNKNOWN/UNKNOWN/UNKNOWN/UNKNOWN';
      my ($projid, $repoid, $arch, $packid) = split('/', $prpap, 4);
      $bin->{'project'} = $projid;
      $bin->{'repository'} = $repoid;
      $bin->{'arch'} = $arch;
      $bin->{'package'} = $packid;
    }
    writexml("$dir/.$reportfile", "$dir/$reportfile", $report, $BSXML::report);
  }
}

sub addcontainerinfodata {
  my ($report, $containerinfofile) = @_;
  return unless -s $containerinfofile && (-s _) < 1000000;
  my $cf;
  if (open($cf, '<', $containerinfofile)) {
    while (<$cf>) {
      $report->{'version'} = $1 if /^\s*\"version\"\s*:\s*\"(.*)\"/;
      $report->{'release'} = $1 if /^\s*\"release\"\s*:\s*\"(.*)\"/;
    }
    close $cf;
  }
}

# create a .report file from a .packages file by adding binary origin information
sub createreport {
  my ($buildinfo, $dir, $kiwiorigins, $srcdir) = @_;

  local *F;
  for my $file (grep {/\.packages$/} sort(ls($dir))) {
    next unless open(F, '<', "$dir/$file");
    # get basepackages if available
    my $bfile = $file;
    $bfile =~ s/\.packages$/.basepackages/;
    my %baseentries;
    my $bf;
    if (open($bf, '<', "$dir/$bfile")) {
      while (<$bf>) {
        chomp;
        my @s = split(/\|/, $_);
        $baseentries{"$s[0]|$s[1]|$s[2]|$s[3]|$s[4]|$s[5]"} = 1;
      }
      close($bf);
    }
    my %summaries;
    for my $sumfile ('.pkgsummaries') {
      my $sfile = $file;
      $sfile =~ s/\.packages$/$sumfile/;
      my $sf;
      if (open($sf, '<', "$dir/$sfile")) {
	while (<$sf>) {
	  chomp;
	  my @s = split(/\|/, $_, 6);
	  $summaries{"$s[0]|$s[1]|$s[2]|$s[3]|$s[4]"} = $s[5];
	}
	close($sf);
	unlink("$dir/$sfile");	# no longer needed
      }
    }

    my @bins;
    while (<F>) {
      chomp;
      my @s = split(/\|/, $_);
      next if @s < 6;
      next if $s[0] eq 'gpg-pubkey';
      my $bin = {
        'name' => $s[0],
        'version' => $s[2],
        'release' => $s[3],
        'binaryarch' => $s[4],
      };
      $bin->{'disturl'} = $s[5] if $s[5] ne '(none)' && $s[5] ne 'None';
      $bin->{'license'} = $s[6] if $s[6];
      $bin->{'package'} = $1 if $s[5] =~ /.*\/[^-]*-(.*)$/; # container case
      $bin->{'epoch'} = $s[1] if $s[1] ne '' && $s[1] ne '(none)' && $s[1] ne 'None';
      if ($s[1] eq 'None' && $s[3] eq 'None') {
	# debian case, split version as kiwi does not do it
	my $evr = $s[2];
	$bin->{'epoch'} = $1 if $evr =~ s/^(\d+)://;
	$bin->{'version'} = $evr;
	$bin->{'release'} = '0';
	if ($evr =~ /^(.+)-([^-]+)$/) {
	  $bin->{'version'} = $1;
	  $bin->{'release'} = $2;
	}
      }
      my $summary = $summaries{"$s[0]|$s[1]|$s[2]|$s[3]|$s[4]"};
      $bin->{'summary'} = $summary if $summary;
      my $isbase = $baseentries{"$s[0]|$s[1]|$s[2]|$s[3]|$s[4]|$s[5]"};
      $bin->{'isbase'} = 1 if $isbase;
      my $prpap = $kiwiorigins->{$s[0]};
      if ($prpap) {
        my ($projid, $repoid, $arch, $packid) = split('/', $prpap, 4);
        if ($buildinfo->{'slsaprovenance'} && !$isbase) {
          my $fn;
          $fn = "$s[0].rpm" if -e "$srcdir/repos/$projid/$repoid/$s[0].rpm";
          $fn = "$s[0].deb" if -e "$srcdir/repos/$projid/$repoid/$s[0].deb";
          add_slsa_materials_binary($buildinfo, "$projid/$repoid/$arch", $fn, "$srcdir/repos/$projid/$repoid/$fn", 'repos') if $fn;
        }
        $bin->{'project'} = $projid;
        $bin->{'repository'} = $repoid;
        $bin->{'arch'} = $arch;
        $bin->{'package'} = $packid if $packid;

        # add cpeid from release rpms only so far for speed reasons
        if ($bin->{'name'} =~ /-release$/) {
          BSVerify::verify_projid($projid);
          BSVerify::verify_repoid($repoid);
          BSVerify::verify_packid($bin->{'name'}); # not exactly, but hopefully good enough
          my $rpm_file = "$srcdir/repos/$projid/$repoid/$bin->{'name'}.rpm";
          if (-e $rpm_file) {
            my $cpeid = cpeid_from_rpm($rpm_file);
            $bin->{'cpeid'} = $cpeid if $cpeid;
          }
        }
      }
      push @bins, $bin;
    }
    close F;
    push @bins, @{$kiwiorigins->{':basecontainers'} || []};
    my $report = { 'binary' => \@bins };
    addreportdata($report, $buildinfo);
    my $containerinfofile = $file;
    $containerinfofile =~ s/\.packages$/\.containerinfo/;
    $containerinfofile =~ s/\.containerinfo$/\.docker\.containerinfo/ unless -e "$dir/$containerinfofile";
    addcontainerinfodata($report, "$dir/$containerinfofile") if -s "$dir/$containerinfofile";
    my $reportfile = $file;
    $reportfile =~ s/\.packages$/\.report/;
    writexml("$dir/.$reportfile", "$dir/$reportfile", $report, $BSXML::report);
  }
}

sub normalize_container {
  my ($container, $recompress, $tmpdir) = @_;
  my ($tar, $mtime, undef, undef, $layercomp) = BSContar::normalize_container($container, $recompress, undef, $tmpdir);
  if (grep {$_ && $_ ne 'gzip'} @{$layercomp || []}) {
    die("non-gzip layer compression, not normalizing container\n");
  }
  BSTar::writetarfile("$container.new", undef, $tar, 'mtime' => $mtime);
  if (-f "$container.sha256") {
    open(NEWTAR, '<', "$container.new") || die("$container.new: $!\n");
    my $ctx = Digest::SHA->new(256);
    my $containerbasename = $container;
    $containerbasename =~ s/.*\///;
    my $chksum = $ctx->addfile(\*NEWTAR)->hexdigest() . "  $containerbasename\n";
    close NEWTAR;
    writestr("$container.sha256.new", "$container.sha256", $chksum);
    utime($mtime, $mtime, "$container.sha256");
  }
  rename("$container.new", "$container") || die("rename $container.new $container: $!\n");
}

# this works for both v02 and v1 predicates
my $slsa_json_template = {
  '_order' => [ '_type', 'subject', 'predicateType', 'predicate' ],
  'subject' => {
    '_order' => [ 'name', 'digest' ],
  },
  'predicate' => {
    '_order' => [ 'buildDefinition', 'runDetails', 'builder', 'buildType', 'invocation', 'buildConfig', 'metadata', 'materials' ],
    # v1
    'buildDefinition' => {
      '_order' => [ 'buildType', 'externalParameters', 'internalParameters', 'resolvedDependencies' ],
      'resolvedDependencies' => {
	'_order' => [ 'name', 'uri', 'digest', 'content', 'downloadLocation', 'mediaType', 'annotations' ],
      }
    },
    'runDetails' => {
      '_order' => [ 'builder', 'metadata', 'byproducts' ],
      'builder' => {
	'_order' => [ 'id', 'builderDependencies', 'version' ],
      },
      'metadata' => {
	'_order' => [ 'invocationId', 'startedOn', 'finishedOn' ],
      },
    },
    # v02
    'invocation' => {
      '_order' => [ 'configSource', 'parameters', 'environment' ],
      'configSource' => {
	'_order' => [ 'uri', 'digest', 'entryPoint' ],
      }
    },
    'metadata' => {
      '_order' => [ 'buildInvocationId', 'buildStartedOn', 'buildFinishedOn', 'completeness', 'reproducible' ],
      'completeness' => {
	'_order' => [ 'parameters', 'environment', 'materials' ],
	'*' => 'bool',
      },
      'reproducible' => 'bool',
    },
    'materials' => {
      '_order' => ['uri', 'digest', 'intent' ],
    }
  }
};

sub provenance_tojson {
  my ($provenance) = @_;
  require Build::SimpleJSON;
  return Build::SimpleJSON::unparse($provenance,  'template' => $slsa_json_template, 'keepspecial' => 1);
}

# Generate subject from the output artifacts
sub generate_slsa_provenance_subject {
  my ($send) = @_;
  my $subject = [];
  for my $artifact (@$send) {
    next if $artifact->{name} =~ /^(rpmlint\.log|_statistics|_buildenv)$/;
    my $sha256sum = sha256file($artifact->{'filename'});
    push @$subject, { 'name' => $artifact->{'name'}, 'digest' => { 'sha256' => $sha256sum }};
  }
  return $subject;
}

sub generate_slsa_sourceuri {
  my ($buildinfo, $filename, $sha256sum) = @_;

  my $filename_suf = defined($filename) ? "/$filename" : '';
  my $uri;
  my $packid = $buildinfo->{'package'};
  $packid =~ s/:.*// if $packid =~ /(?<!^_product)(?<!^_patchinfo):./;
  if ($buildinfo->{'slsadownloadurl'} && $buildinfo->{'slsadownloadurl'} =~ /(.*)\/_slsa$/) {
    # using the source publishing server
    my $server = $1;
    $uri = BSHTTP::urlencode("$server/$buildinfo->{'project'}/$packid/$buildinfo->{'srcmd5'}$filename_suf");
  } else {
    my $server = $buildinfo->{'slsadownloadurl'} || $buildinfo->{'srcserver'} || $srcserver;
    $uri = BSHTTP::urlencode("$server/source/$buildinfo->{'project'}/$packid$filename_suf")."?rev=$buildinfo->{'srcmd5'}";
  }
  return $uri;
}

sub generate_slsa_binaryuri {
  my ($buildinfo, $prpa, $filename, $sha256sum) = @_;
  my $uri;
  if ($buildinfo->{'slsadownloadurl'} && $buildinfo->{'slsadownloadurl'} =~ /\/_slsa$/) {
    # using the source publishing server
    my $server = $buildinfo->{'slsadownloadurl'};
    $uri = BSHTTP::urlencode("$server/$prpa/$filename/$sha256sum");
  } else {
    # using the api server
    my $server = $buildinfo->{'slsadownloadurl'} || $buildinfo->{'srcserver'} || $srcserver;
    $uri = BSHTTP::urlencode("$server/slsa/$prpa/$filename/$sha256sum");
  }
  return $uri;
}

sub generate_slsa_provenance_statement_v02 {
  my ($buildinfo, $send) = @_;

  my $subject = generate_slsa_provenance_subject($send);
  my $sourceuri = generate_slsa_sourceuri($buildinfo);
  # strip name and intent if not containers/repos
  my @materials = map { { %$_ } } @{$buildinfo->{'materials'} || []};
  for (@materials) {
    delete $_->{'name'};
    delete $_->{'intent'} if $_->{'intent'} && $_->{'intent'} ne 'containers' && $_->{'intent'} ne 'repos' && $_->{'intent'} ne 'sysroot';
  }
  my $stmt = {
    '_type' => "https://in-toto.io/Statement/v0.1",
    'subject' => $subject,
    'predicateType' => 'https://slsa.dev/provenance/v0.2',
    'predicate' => {
      'builder' => {
        'id' => $buildinfo->{'slsabuilder'}
      },
      'buildType' => 'https://open-build-service.org/worker',
      'invocation' => {
        'configSource' => {
          'uri' => $sourceuri,
          'entryPoint' => $buildinfo->{'file'},
        },
      },
      'metadata' => {
        'completeness' => {
          'parameters' => 1,
          'environment' => 1,
          'materials' => 1,
        },
        'reproducible' => 0,
      },
      'materials' => \@materials,
    }
  };
  $stmt->{'predicate'}->{'invocation'}->{'parameters'}->{'release'} = $buildinfo->{'release'} if defined $buildinfo->{'release'};
  $stmt->{'predicate'}->{'invocation'}->{'parameters'}->{'buildflavor'} = $buildinfo->{'buildflavor'} if defined $buildinfo->{'buildflavor'};
  return provenance_tojson($stmt);
}

sub generate_slsa_provenance_statement_v1 {
  my ($buildinfo, $send) = @_;

  # generate flags for buildenv binaries
  my $imagemode = getimagemode($buildinfo);
  my @bdep = @{$buildinfo->{'bdep'} || []};
  my %flags;
  $flags{$_->{'name'}}->{'preinstall'} = 1 for grep {$_->{'preinstall'}} @bdep;
  $flags{$_->{'name'}}->{'vminstall'} = 1 for grep {$_->{'vminstall'}} @bdep;
  $flags{$_->{'name'}}->{'runscripts'} = 1 for grep {$_->{'runscripts'}} @bdep;
  if (!$imagemode) {
    $flags{$_->{'name'}}->{'noinstall'} = 1 for grep {$_->{'noinstall'}} @bdep;
    $flags{$_->{'name'}}->{'installonly'} = 1 for grep {$_->{'installonly'}} @bdep;
  }

  my $subject = generate_slsa_provenance_subject($send);
  my $sourceuri = generate_slsa_sourceuri($buildinfo);
  # make intent an annotation
  my @materials = map { { %$_ } } @{$buildinfo->{'materials'} || []};
  for (@materials) {
    my $i = delete $_->{'intent'};
    $_->{'annotations'}->{'intent'} = $i if $i;
    if ($i && $i eq 'buildenv') {
      my $binname;
      $binname = $1 if $_->{'name'} =~ /(.*)\.(?:$binsufsre)$/;
      $_->{'annotations'}->{'flags'} = join(',', sort keys %{$flags{$binname}}) if $binname && $flags{$binname};
    }
  }
  my $stmt = {
    '_type' => "https://in-toto.io/Statement/v0.1",
    'subject' => $subject,
    'predicateType' => 'https://slsa.dev/provenance/v1',
    'predicate' => {
      'buildDefinition' => {
	'buildType' => 'https://open-build-service.org/worker',
	'externalParameters' => {
	  'source' => $sourceuri,
	  'recipeFile' => $buildinfo->{'file'},
	},
	'resolvedDependencies' => \@materials,
      },
      'runDetails' => {
	'id' => $buildinfo->{'slsabuilder'},
      },
    }
  };
  my $external_params = $stmt->{'predicate'}->{'buildDefinition'}->{'externalParameters'};
  $external_params->{'release'} = $buildinfo->{'release'} if defined $buildinfo->{'release'};
  $external_params->{'buildflavor'} = $buildinfo->{'buildflavor'} if defined $buildinfo->{'buildflavor'};
  $external_params->{'disturl'} = $buildinfo->{'provenance_disturl'} if defined $buildinfo->{'provenance_disturl'};
  $external_params->{'vcs'} = $buildinfo->{'provenance_vcs'} if defined $buildinfo->{'provenance_vcs'};
  return provenance_tojson($stmt);
}

sub add_slsa_materials_sourcefile {
  my ($buildinfo, $filename, $localfile) = @_;
  my $sha256sum = sha256file($localfile);
  my $uri = generate_slsa_sourceuri($buildinfo, $filename, $sha256sum);
  push @{$buildinfo->{'materials'}}, { 'name' => $filename, 'uri' => $uri, 'digest' => { 'sha256' => $sha256sum }, 'intent' => 'source' };
}

sub add_slsa_materials_binary {
  my ($buildinfo, $prpa, $filename, $localfile, $intent) = @_;
  my $sha256sum = sha256file($localfile);
  my $uri = generate_slsa_binaryuri($buildinfo, $prpa, $filename, $sha256sum);
  push @{$buildinfo->{'materials'}}, { 'name' => $filename, 'uri' => $uri, 'digest' => { 'sha256' => $sha256sum }, 'intent' => ($intent || 'buildenv') };
}

sub add_slsa_materials_config {
  my ($buildinfo, $prp, $config) = @_;
  my $sha256sum = Digest::SHA::sha256_hex($config);
  my ($prpa, $filename) = ("$prp/noarch", '_config');
  my $uri = generate_slsa_binaryuri($buildinfo, $prpa, $filename, $sha256sum);
  push @{$buildinfo->{'materials'}}, { 'name' => $filename, 'uri' => $uri, 'digest' => { 'sha256' => $sha256sum }, 'intent' => 'buildconfig' };
}

$Build::Kiwi::urlmapper = sub {return '_obsrepositories'}  unless $Build::Kiwi::urlmapper;

sub dobuild {
  my ($buildinfo) = @_;

  my $projid = $buildinfo->{'project'};
  my $packid = $buildinfo->{'package'};
  my $repoid = $buildinfo->{'repository'};
  my $arch = $buildinfo->{'arch'};
  my $helperarch = $buildinfo->{'hostarch'} || $arch;
  my $imagemode = getimagemode($buildinfo);
  my $kiwiorigins;
  my $stats = {};
  my $deltamode = $buildinfo->{'file'} eq '_delta' ? 1 : undef;
  my $ptfmode = $buildinfo->{'file'} eq '_ptf' ? 1 : undef;
  my $followupmode = $buildinfo->{'followupfile'};
  my $followupcopy = {};
  my $needsobspackage;

  my $helper = '';
  /^\Q$helperarch\E:(.*)$/ && ($helper = $1) for @{$BSCando::cando{$hostarch}};

  my $starttime = time();
  print BSUtil::isotime($starttime).": building '$packid' for project '$projid' repository '$repoid' arch '$arch'";
  print " using helper $helper" if $helper;
  print "\n";

  my $srcdir    = "$buildroot/.build-srcdir";
  my $pkgdir    = "$buildroot/.pkgs";
  my $oldpkgdir = "$buildroot/.build.oldpackages";

  if ($vm_tmpfs_mode) {
    my @targs;
    # Note that the xen/kvm vmdisk_rootsize is in MB
    push @targs, "mount", "-t", "tmpfs", "-osize=${vmdisk_rootsize}M", "none", $buildroot;
    qsystem(@targs) && die("mount tmpfs failed: $!\n");
  }

  unlink("$buildroot/.build.meta");
  unlink("$buildroot/.build.packages");
  rm_rf("$buildroot/.build.packages") if -d "$buildroot/.build.packages";
  rm_rf($srcdir) if -d $srcdir;
  # changed to cleandir so that pkgdir can be a symlink
  BSUtil::cleandir($pkgdir) if -d $pkgdir;
  rm_rf($oldpkgdir) if -d $oldpkgdir;

  # create buildenv data
  if (!$imagemode && !$followupmode && !$deltamode && !$ptfmode) {
    $buildinfo->{'outbuildinfo'} = {
      'project' => $projid,
      'package' => $packid,
      'repository' => $repoid,
      'arch' => $arch,
      'srcmd5' => $buildinfo->{'srcmd5'},
      'verifymd5' => $buildinfo->{'verifymd5'} || $buildinfo->{'srcmd5'},
      'bdep' => [],
    };
    for ('versrel', 'bcnt', 'release', 'module', 'crossarch', 'signflavor') {
      $buildinfo->{'outbuildinfo'}->{$_} = $buildinfo->{$_} if defined $buildinfo->{$_};
    }
  }

  ####### download phase
  $binariesdownload = 0;
  $binariescachehits = 0;
  $binariesdownloadsize = 0;
  my $downloadstarttime = time();
  my @meta;
  my $preinstallimagedata = {};
  print "fetching sources, ";
  mkdir($srcdir) || die("mkdir $srcdir: $!\n");
  if ($deltamode) {
    push @meta, ($buildinfo->{'verifymd5'} || $buildinfo->{'srcmd5'})."  $buildinfo->{'package'}";
    getjobdata($buildinfo, $srcdir);
    print "packages, ";
    getbinaries($buildinfo, $pkgdir, $srcdir, $preinstallimagedata);
    undef $oldpkgdir;
    if (! -s "$srcdir/deltagen.spec") {
      my $spec = readstr("$statedir/worker/worker-deltagen.spec", 1) || readstr("worker-deltagen.spec");
      writestr("$srcdir/deltagen.spec", undef, $spec);
    }
    $buildinfo->{'file'} = 'deltagen.spec';
  } elsif ($ptfmode) {
    getjobdata($buildinfo, $srcdir);
    print "packages, ";
    getbinaries($buildinfo, $pkgdir, $srcdir, $preinstallimagedata);
    undef $oldpkgdir;
    $buildinfo->{'file'} = 'ptf.spec';
    @meta = split("\n", readstr("$srcdir/meta"));
  } elsif ($followupmode) {
    getfollowupsources($buildinfo, $srcdir);
    readbuildenv($buildinfo, $srcdir);
    getsslcert($buildinfo, $srcdir);
    unlink("$srcdir/_buildenv.$buildinfo->{'repository'}.$buildinfo->{'arch'}");
    print "packages, ";
    if ($imagemode && ($imagemode eq 'kiwiproduct' || $imagemode eq 'productcompose')) {
      getbinaries_product($buildinfo, $pkgdir, $srcdir, {});
      $followupcopy->{$_} = readstr("$srcdir/$_") for grep {$_ eq '_channel' || /\.report$/} ls($srcdir);
    } else {
      getbinaries($buildinfo, $pkgdir, $srcdir, $preinstallimagedata);
    }
    undef $oldpkgdir;
    $buildinfo->{'rootforbuild'} = 1 if $buildinfo->{'file'} =~ /\.kiwi$/;
    $buildinfo->{'file'} = $followupmode;
    # recalc kiwimode as we changed the file
    $imagemode = getimagemode($buildinfo);
    @meta = split("\n", readstr("$srcdir/meta"));
  } else {
    $kiwiorigins = {} if $imagemode;	# always for image builds
    my $needspubkey;
    my $needssslcert;
    my $needsappxsslcert;
    my $needsbinariesforbuild;
    push @meta, getsources($buildinfo, $srcdir);
    importbuild() unless defined &Build::queryhdrmd5;
    $buildinfo->{'buildtype'} = Build::recipe2buildtype($buildinfo->{'file'});
    if (!$imagemode && -s "$srcdir/$buildinfo->{'file'}") {
      # check for build markers
      local *F;
      if (open(F, '<', "$srcdir/$buildinfo->{'file'}")) {
	while(<F>) {
	  chomp;
	  $needsbinariesforbuild = 1 if /^#\s*needsbinariesforbuild\s*$/s;
	  $needspubkey = 1 if /^#\s*needspubkeyforbuild\s*$/s;
	  $needssslcert = 1 if /^#\s*needssslcertforbuild\s*$/s;
	  # On Debian we cannot add a commented out line in a dsc, use a tag instead
	  # that can be generated by adding "XS-Obs: needssslcertforbuild" to debian/control
	  $needssslcert = 1 if /^Obs:\s*needssslcertforbuild\s*$/s;
	  $needsappxsslcert = 1 if /^(?:#|Obs:)\s*needsappxsslcertforbuild\s*$/s;
	  $needsobspackage = 1 if /\@OBS_PACKAGE\@/;
	}
	close F;
      }
    }
    my $buildtype = $buildinfo->{'buildtype'} || '';
    $needsbinariesforbuild = 1 if $buildtype eq 'mkosi' || $buildtype eq 'livebuild' || $buildtype eq 'appimage' || $buildtype eq 'simpleimage';
    if ($needsbinariesforbuild) {
      undef $preinstallimagedata;		# can't use a preinstall image, sorry
      $kiwiorigins = {};			# but collect origins
    }
    $buildinfo->{'forcebinaryidmeta'} = 1 if $buildtype eq 'mkosi' || $buildtype eq 'livebuild' || $buildtype eq 'appimage' || $buildtype eq 'simpleimage' || $buildtype eq 'preinstallimage';
    getobsgendiffdata($buildinfo, $srcdir) if $buildinfo->{'obsgendiff'};
    readbuildenv($buildinfo, $srcdir);
    getsslcert($buildinfo, $srcdir, $needsappxsslcert ? 'appx' : undef) if $needssslcert || $needsappxsslcert;
    getpubkey($buildinfo, $srcdir) if $needspubkey;
    print "packages, ";
    if (($imagemode || '') eq 'kiwiproduct') {
      $buildinfo->{'rpmhdrs_only'} = 1 if kiwiproduct_is_rpmhdrsonly("$srcdir/$buildinfo->{'file'}");
      push @meta, getbinaries_product($buildinfo, $pkgdir, $srcdir, $kiwiorigins);
    } elsif (($imagemode || '') eq 'productcompose') {
      push @meta, getbinaries_product($buildinfo, $pkgdir, $srcdir, $kiwiorigins);
    } else {
      push @meta, getbinaries($buildinfo, $pkgdir, $srcdir, $preinstallimagedata, $kiwiorigins);
    }
  }
  $preinstallimagedata ||= {};
  undef $oldpkgdir if $buildinfo->{'nounchanged'};

  writestr("$buildroot/.build.meta", undef, join("\n", @meta)."\n");

  my @configpath;
  if ($imagemode) {
    @configpath = map {"path=$_->{'project'}/$_->{'repository'}"} @{$buildinfo->{'syspath'} || $buildinfo->{'path'} || []};
    unshift @configpath, "path=$projid/$repoid" unless @configpath && $configpath[0] eq "path=$projid/$repoid";
  }
  my $server = $buildinfo->{'srcserver'} || $srcserver;
  my $config;
  if (defined $buildinfo->{'config'}) {
    $config = $buildinfo->{'config'};
  } else {
    $config = BSRPC::rpc("$server/getconfig", undef, "project=$projid", "repository=$repoid", @configpath);
    $config = Build::combine_configs($config, "Macros:\n$buildinfo->{'modularity_macros'}") if $buildinfo->{'modularity_macros'};
  }
  $buildinfo->{'slsa_provenance_config'} = $config if $buildinfo->{'slsaprovenance'};
  $buildinfo->{'outbuildinfo'}->{'config'} = $config;
  writestr("$buildroot/.build.config", undef, $config);
  add_slsa_materials_config($buildinfo, "$projid/$repoid", $config) if $buildinfo->{'slsaprovenance'};

  my $bconf = Build::read_config($buildinfo->{'arch'}, "$buildroot/.build.config");
  my $useccache = $buildinfo->{'ccache'} ? 1 : 0;
  my $ccachetype = $useccache ? $buildinfo->{'ccache'} : undef;
  $buildinfo->{'slsa_provenance_version'} = $bconf->{'buildflags:slsaversion'} if $buildinfo->{'slsaprovenance'};

  if ($buildinfo->{'modularity_meta'}) {
    getmodulemddata($buildinfo, $srcdir, $bconf);
  }

  if ($oldpkgdir && !$imagemode && $buildinfo->{'file'} ne 'preinstallimage') {
    getoldpackages($buildinfo, $oldpkgdir, $useccache);
    getlinkedccache($buildinfo, $oldpkgdir) if $useccache && ! -f "$oldpkgdir/_ccache.tar";
  }

  $stats->{'times'}->{'download'}->{'time'}->{'unit'} = "s";
  $stats->{'times'}->{'download'}->{'time'}->{'_content'} = (time() - $downloadstarttime);
  $stats->{'download'}->{'cachehits'}          = $binariescachehits;
  if ($binariesdownload) {
    $stats->{'download'}->{'size'}->{'unit'} = 'k';
    $stats->{'download'}->{'size'}->{'_content'} = $binariesdownloadsize;
    $stats->{'download'}->{'binaries'}           = $binariesdownload;
  }
  $stats->{'download'}->{'preinstallimage'} = $preinstallimagedata->{'imagename'} if $preinstallimagedata->{'imagename'};

  ####### end of download phase
  $buildinfo->{'download_complete'} = 1;

  if ($deltamode) {
    # replace @mopt@ in specfile
    my $spec = readstr("$srcdir/deltagen.spec");
    if ($spec =~ /\@mopt\@/s) {
      my $mopt;
      $mopt = $BSConfig::deltagen_mopt if defined $BSConfig::deltagen_mopt;
      for (@{$bconf->{'repotype'}}) {
	$mopt = $1 eq '' ? undef : $1 if /^deltagen-mopt:(\d*)/;
      }
      $mopt = 512 unless defined $mopt;
      $mopt = '' unless $mopt;
      $spec =~ s/\@mopt\@/$mopt/sg;
      writestr("$srcdir/deltagen.spec", undef, $spec);
    }
  }

  my $release = $buildinfo->{'release'};
  my $disturl;
  $disturl = $buildinfo->{'disturl'} if $buildinfo->{'disturl'} && !$bconf->{'buildflags:nodisturl'};
  my $vcs;
  $vcs = $buildinfo->{'vcs'} if $buildinfo->{'vcs'} && $bconf->{'buildflags:setvcs'};

  $buildinfo->{'provenance_disturl'} = $disturl if $disturl;
  $buildinfo->{'provenance_vcs'} = $vcs if $vcs;

  my @args;
  push @args, $helper if $helper;

  # build rpmlist for build script
  my @rpmlist;
  my @bdep = @{$buildinfo->{'bdep'} || []};
  my $imagebins = $preinstallimagedata->{'imagebins'} || {};
  for my $bdep (@bdep) {
    next if $bdep->{'package'} || (($bdep->{'repoarch'} || '') eq 'src');
    next if $imagemode && $bdep->{'noinstall'};
    my $bin = $bdep->{'name'};
    if ($bdep->{'sysroot'}) {
      $bin = "__sysroot__$bin";
    } elsif ($imagebins->{$bin}) {
      push @rpmlist, "$bin preinstallimage";
      next;
    }
    for my $osuf (@binsufs, '__notfound') {
      die("missing package: $bin\n") if $osuf eq '__notfound';
      next unless -e "$pkgdir/$bin.$osuf";
      if ($bdep->{'sysroot'}) {
        push @rpmlist, "sysroot: $bdep->{'name'} $pkgdir/$bin.$osuf";
      } else {
        push @rpmlist, "$bin $pkgdir/$bin.$osuf";
      }
      last;
    }
  }
  push @rpmlist, "preinstallimage: $pkgdir/$preinstallimagedata->{'imagename'}" if $preinstallimagedata->{'imagename'};
  push @rpmlist, "preinstallimagesource: $preinstallimagedata->{'imagesource'}" if $preinstallimagedata->{'imagesource'};
  push @rpmlist, "localkiwi $localkiwi/localkiwi.rpm" if $localkiwi && -e "$localkiwi/localkiwi.rpm";
  push @rpmlist, "preinstall: ".join(' ', map {$_->{'name'}} grep {$_->{'preinstall'}} @bdep);
  push @rpmlist, "vminstall: ".join(' ', map {$_->{'name'}} grep {$_->{'vminstall'}} @bdep);
  push @rpmlist, "runscripts: ".join(' ', map {$_->{'name'}} grep {$_->{'runscripts'}} @bdep);
  if (!$imagemode) {
    push @rpmlist, "noinstall: ".join(' ', map {$_->{'name'}} grep {$_->{'noinstall'}} @bdep);
    push @rpmlist, "installonly: ".join(' ', map {$_->{'name'}} grep {$_->{'installonly'}} @bdep);
  }
  writestr("$buildroot/.build.rpmlist", undef, join("\n", @rpmlist)."\n");

  print "building...\n";

  die("$buildinfo->{'error'}\n") if $buildinfo->{'error'};
  if (($imagemode || '') eq 'kiwiproduct') {
    kiwiproduct_patch_recipe($buildinfo, "$srcdir/$buildinfo->{'file'}", \@meta);
  }

  push @args, "$statedir/build/build";
  if ($vm =~ /(xen|kvm|qemu|zvm|emulator|pvm)/) {
    # allow overriding the filesystem type via the build config
    my $filesystemtype = $bconf->{'buildflags:vmfstype'} || $vmdisk_filesystem;
    my $filesystemoptions = $bconf->{'buildflags:vmfsoptions'} || $vmdisk_filesystem_options;
    mkdir("$buildroot/.mount") unless -d "$buildroot/.mount";
    push @args, '--root', "$buildroot/.mount";
    push @args, '--vm-type', $vm;
    push @args, '--vm-disk', $vm_root;
    push @args, '--vm-swap', $vm_swap;
    push @args, '--emulator-script', $emulator_script if $vm eq 'emulator' && $emulator_script;
    push @args, '--statistics';
    push @args, '--vm-watchdog';
    my $vmmemory = $vm_memory ? $vm_memory : readstr("$buildroot/memory", 1);
    chomp $vmmemory if $vmmemory;
    push @args, '--memory', $vmmemory if $vmmemory;
    push @args, '--vm-kernel', $vm_kernel if $vm_kernel;
    push @args, '--vm-initrd', $vm_initrd if $vm_initrd;
    push @args, "--vm-custom-opt=$vm_custom_option" if $vm_custom_option;
    push @args, '--vmdisk-rootsize', $vmdisk_rootsize if $vmdisk_rootsize;
    push @args, '--vmdisk-swapsize', $vmdisk_swapsize if $vmdisk_swapsize;
    push @args, '--vmdisk-filesystem', $filesystemtype if $filesystemtype;
    push @args, '--vmdisk-filesystem-options', $filesystemoptions if $filesystemoptions;
    push @args, '--vm-use-mkfs-copyin' if $vm_use_mkfs_copyin && $filesystemtype =~ /^ext/;
    push @args, "--vmdisk-mount-options=$vmdisk_mount_options" if $vmdisk_mount_options;
    push @args, '--vmdisk-clean'if $vmdisk_clean;
    push @args, '--hugetlbfs', $hugetlbfs if $hugetlbfs;
    push @args, '--vm-worker', $vm_worker_name if $vm_worker_name;
    push @args, '--vm-worker-nr', $vm_worker_instance if $vm_worker_instance;
    push @args, '--vm-network' if $vm eq 'kvm' && $vm_network;
    push @args, '--vm-enable-console' if $vm_enable_console;
  } elsif ($vm eq 'openstack') {
    mkdir("$buildroot/.mount") unless -d "$buildroot/.mount";
    push @args, '--root',      "$buildroot/.mount";
    push @args, "--vm-type",   $vm;
    push @args, "--vm-disk",   $vm_root;
    push @args, "--vm-swap",   $vm_swap;
    push @args, "--vm-server", $vm_server;
    push @args, "--vm-worker", $vm_worker_name;
    push @args, "--vm-kernel", $vm_kernel;
    push @args, "--openstack-flavor", "$openstack_flavor";
  } elsif ($vm eq 'lxc' || $vm eq 'docker') {
    push @args, '--root', $buildroot;
    push @args, '--vm-type', $vm;
    push @args, '--memory', $vm_memory if $vm_memory;
  } else {
    print "VM-TYPE $vm not detected" if $vm;
    push @args, '--root', $buildroot;
    push @args, '--vm-type', $vm if $vm;
  }
  push @args, '--clean';
  push @args, '--changelog';
  push @args, '--oldpackages', $oldpkgdir if $oldpkgdir && -d $oldpkgdir;
  push @args, '--norootforbuild' unless !$BSConfig::norootexceptions || $bconf->{'buildflags:allowrootforbuild'} || $buildinfo->{'rootforbuild'} || grep {"$projid/$packid" =~ /^$_$/} keys %$BSConfig::norootexceptions;
  push @args, '--baselibs-internal';
  push @args, '--dist', "$buildroot/.build.config";
  push @args, '--rpmlist', "$buildroot/.build.rpmlist";
  push @args, '--logfile', "$buildroot/.build.log";
  push @args, '--release', "$release" if defined $release;
  push @args, '--debug' if $buildinfo->{'debuginfo'};
  push @args, '--arch', $arch;
  push @args, '--jobs', $jobs if $jobs;
  push @args, '--ccache' if $useccache && $oldpkgdir;
  push @args, '--ccache-create-archive' if $useccache && $oldpkgdir;
  push @args, "--ccache-type=$ccachetype" if $useccache && $oldpkgdir && $ccachetype;
  push @args, '--threads', $threads if $threads;
  push @args, '--reason', "Building $packid for project '$projid' repository '$repoid' arch '$arch' srcmd5 '$buildinfo->{'srcmd5'}'";
  push @args, '--disturl', $disturl if $disturl;
  push @args, '--linksources' if $localkiwi;
  push @args, '--signdummy' if $imagemode && ($imagemode eq 'kiwiproduct' || $imagemode eq 'productcompose') && -e "$statedir/build/signdummy";
  push @args, '--vcs', $vcs if $vcs;
  if ($packid =~ /(?<!^_product)(?<!^_patchinfo):./) {
    $packid =~ /^(.*):(.*?)$/;
    push @args, "--obspackage=$1" if $needsobspackage;
    push @args, "--buildflavor=$2";
    $buildinfo->{'buildflavor'} = $2;
  } else {
    push @args, "--obspackage=$packid" if $needsobspackage;
  }
  push @args, "$srcdir/$buildinfo->{'file'}";
  if (($imagemode || '') eq 'kiwiimage') {
    # expand obsrepositories:/ directive for appliances
    my $kiwi = Build::parse(undef, "$srcdir/$buildinfo->{'file'}") || {};
    if (grep {$_->{'project'} eq '_obsrepositories'} @{$kiwi->{'path'} || []}) {
      push @args, '--kiwi-parameter', '--ignore-repos';
      my $prio = 0;
      for my $prjpath (@{$buildinfo->{'path'} || []}) {
        $prio = $prio + 1;
	my $repo ="dir://./repos/$prjpath->{'project'}/$prjpath->{'repository'}";
	$repo =~ s/:/:\//g;		# XXX: no longer needed?
	push @args, '--kiwi-parameter', '--add-repo', '--kiwi-parameter', $repo;
	push @args, '--kiwi-parameter', '--add-repotype', '--kiwi-parameter', 'rpm-md';
	push @args, '--kiwi-parameter', '--add-repoprio', '--kiwi-parameter', $prio;	# different prio for smart?
      }
    }
    if (@{$buildinfo->{'containerurl'} || []}) {
      push @args, '--kiwi-parameter', '--set-container-derived-from='.join(',', @{$buildinfo->{'containerurl'}});
    }
  }
  if ($imagemode && $imagemode ne 'kiwiproduct' && $imagemode ne 'productcompose') {
    # touch pkgsummaries marker so we get summaries in the report files
    mkdir_p("$srcdir/repos");
    BSUtil::touch("$srcdir/repos/.pkgsummaries");
  }
  qsystem(@args);

  print "\n";
  print BSUtil::isotime(time()).": finished '$packid' for project '$projid' repository '$repoid' arch '$arch'";
  print " using helper $helper" if $helper;
  print "\n";

  my $ret = $?;

  my $resultdir = "$buildroot/.build.packages";
  $resultdir = "$buildroot/.mount/.build.packages" if $vm =~ /(xen|kvm|qemu|zvm|emulator|pvm|openstack)/;

  # change exit code to unchanged if the same_result_marker is present
  if ($ret == 0 && ($buildinfo->{'reason'} || '') ne 'rebuild counter sync') {
    $ret = 2 * 256 if -e "$resultdir/same_result_marker";
  }

  # ignore unchanged exit code if we do a rebuild counter sync build
  $ret = 0 if $ret == 2 * 256 && ($buildinfo->{'reason'} || '') eq 'rebuild counter sync';

  # update the statistics with vm data if the build succeeded or is unchanged
  if (($ret == 0 || $ret == 2 * 256) && $vm =~ /(xen|kvm|qemu|zvm|emulator|pvm|openstack)/) {
    my $statsfile = "$resultdir/OTHER/_statistics";
    updatestatsfromfile($stats, $statsfile) if -e $statsfile;
  }

  # rename build result to final location for succeeded/dynamic buildrequires changed
  if (($ret == 0 || $ret == 9 * 256) && $resultdir ne "$buildroot/.build.packages") {
    rm_rf("$buildroot/.build.packages");
    if (!rename($resultdir, "$buildroot/.build.packages")) {
      print "final rename failed: $!\n";
      return 1;
    }
  }
  $resultdir = "$buildroot/.build.packages";

  # write statistics for succeeded/unchanged builds
  if ($ret == 0 || $ret == 2 * 256) {
    my $now = time();
    $stats->{'times'}->{'total'}->{'time'} = { 'unit' => 's', '_content' => ($now - $starttime) };
    # add some extra data
    my $info = {
      'hostarch' => $hostarch,
      'starttime' => $starttime,
      'endtime' => $now,
    };
    $info->{'workerid'} = $workerid if defined $workerid;
    for (qw{readytime srcmd5 verifymd5 rev vcs file buildtype}) {
      $info->{$_} = $buildinfo->{$_} if $buildinfo->{$_};
    }
    $stats->{'info'} = $info;
    mkdir_p("$resultdir/OTHER");
    writexml("$resultdir/OTHER/_statistics.new", "$resultdir/OTHER/_statistics", $stats, $BSXML::buildstatistics);
  }

  if ($ret == 2 * 256) {		# exit code 2: unchanged
    return 2;
  } elsif ($ret == 3 * 256) {
    return 3;
  } elsif ($ret == 4 * 256) {
    return 4;
  } elsif ($ret == 9 * 256) {		# exit code 9: dynamic buildrequires changed
    return -e "$resultdir/OTHER/_statistics" ? 9 : 1;
  }
  if ($ret) {
    return 1;
  }
  if (! -s "$buildroot/.build.log") {
    print "build succeeded, but no logfile?\n";
    return 1;
  }

  if ($buildinfo->{'outbuildinfo'}) {
    writexml("$resultdir/OTHER/_buildenv", undef, $buildinfo->{'outbuildinfo'}, $BSXML::buildinfo);
  }

  if (!$followupmode && $kiwiorigins) {
    $buildinfo->{'buildtime'} = time();
    if (!$imagemode) {
      # some builds like python-venv also write a .packages file
      createreport($buildinfo, "$resultdir/OTHER", $kiwiorigins, $srcdir);
    } elsif ($imagemode eq 'docker') {
      createreport($buildinfo, "$resultdir/DOCKER", $kiwiorigins, $srcdir);
    } elsif ($imagemode eq 'kiwiproduct') {
      # as a special service we also create a channel file from the report files
      createchannel($buildinfo, "$resultdir/OTHER", $kiwiorigins, $srcdir);
    } elsif ($imagemode eq 'productcompose') {
      annotatereport($buildinfo, "$resultdir/PRODUCT", $kiwiorigins);
    } else {
      createreport($buildinfo, "$resultdir/KIWI", $kiwiorigins, $srcdir);
    }
  }

  if (%$followupcopy) {
    writestr("$resultdir/OTHER/$_", undef, $followupcopy->{$_}) for keys %$followupcopy;
  }
  return 0;
}


sub updatestatsfromfile {
  my ($stats, $filepath) = @_;
  my $iosectors = 0;
  my $iorequests = 0;
  open(FILE, "<", $filepath) || die;
  my %timekeys = (
    'TIME_preinstall' => 'preinstall',
    'TIME_install' => 'install',
    'TIME_main_build' => 'main',
    'TIME_postchecks' => 'postchecks',
    'TIME_rpmlint' => 'rpmlint',
    'TIME_buildcmp' => 'buildcmp',
    'TIME_deltarpms' => 'deltarpms',
    'TIME_cpusteal' => 'cpusteal',
  );
  while (<FILE>) {
    chomp;
    my ($key, $value) = split(": ", $_);
    if ($key eq "MAX_mb_used_on_disk" ){
      $stats->{'disk'}->{'usage'}->{'size'} = { 'unit' => 'M', '_content' => $value };
    } elsif ($key eq "MAX_mb_used_memory" ){
      $stats->{'memory'}->{'usage'}->{'size'} = { 'unit' => 'M', '_content' => $value };
    } elsif ($key eq "IO_requests_read" || $key eq "IO_requests_write" ){
      $iorequests += $value;
      $stats->{'disk'}->{'usage'}->{'io_requests'} = $iorequests;
    } elsif ($key eq "IO_sectors_read" || $key eq "IO_sectors_write" ){
      $iosectors += $value;
      $stats->{'disk'}->{'usage'}->{'io_sectors'} = $iosectors;
    } elsif ($timekeys{$key}) {
      $stats->{'times'}->{$timekeys{$key}}->{'time'} = { 'unit' => 's', '_content' => $value };
    }
  }
  close FILE;
}

sub buildkiwitree {
  my ($destfile, @dirs) = @_;
  my %tree;
  my @todo;
  while (@dirs) {
    my $dir = shift @dirs;
    my $sdir = shift @dirs;
    $sdir = '' unless defined $sdir;
    next unless -d $dir;
    $tree{$1} = "d ". BSHTTP::urlencode($1) if $sdir =~ /^(.+)\/$/;
    push @todo, $dir, $sdir;
  }
  while (@todo) {
    my $dir = shift @todo;
    my $sdir = shift @todo;
    for my $f (sort(ls($dir))) {
      my $sf = "$sdir$f";
      $f = "$dir/$f";
      next if $tree{$sf};
      if (-l $f) {
	my $sl = readlink($f);
	die("readlink $f: $!\n") unless defined $sl;
	$tree{$sf} = "l ". BSHTTP::urlencode($sf) . " " . BSHTTP::urlencode($sl);
      } elsif (-d $f) {
	$tree{$sf} = "d ". BSHTTP::urlencode($sf);
	push @todo, $f, "$sf/";
      } elsif (-f $f) {
	my $extra;
	if ($f =~ /\.slsa_provenance\.json$/) {
	  $extra = md5file($f);
	} elsif ($f =~ /\.rpm$/) {
	  Build::queryhdrmd5($f, \$extra);
	}
	$tree{$sf} = 'f ' . BSHTTP::urlencode($sf) . ($extra ? " $extra" : '');
      }
    }
  }
  local *F;
  open(F, '>', $destfile) || die("$destfile: $!\n");
  for (sort keys %tree) {
    print F "$tree{$_}\n";
  }
  close(F) || die("close $destfile: $!\n");
}

sub getstate {
  my ($jobid, $needbuilding) = @_;
  my $state = readxml("$statedir/state", $BSXML::workerstate, 1);
  $state ||= {'state' => 'unknown state'};
  die("not building a job\n") if ($jobid || $needbuilding) && $state->{'state'} ne 'building';
  die("building a different job\n") if $jobid && $jobid ne $state->{'jobid'};
  return $state;
}

sub changestate {
  my ($jobid, $newstate) = @_;
  my $state = lockstate();
  die("not building a job\n") if !$state || $state->{'state'} ne 'building';
  die("building a different job\n") if $jobid && $jobid ne $state->{'jobid'};
  $state->{'state'} = $newstate;
  commitstate($state);
}

sub append_nonfatal {
  my ($file, $str) = @_;
  local *F;
  if (open(F, '>>', $file)) {
    print F $str;
    close F;
  }
}

sub stdreply {
  my @rep = @_; 
  return unless @rep && defined($rep[0]);
  if (ref($rep[0]) eq 'HASH') {
    if (ref($rep[1]) eq 'CODE') {
      $rep[1] = $rep[1]->($rep[0]);
    } else {
      $rep[1] = BSUtil::toxml($rep[0], $rep[1]);
    }   
    shift @rep;
  }
  push @rep, 'Content-Type: text/xml' if @rep == 1;
  BSServer::reply(@rep);
}

sub errreply {
  my ($err, $code, $tag, @hdrs) = @_; 
  my $opresult = {'code' => $code, 'summary' => $tag};
  $opresult->{'details'} = $err if $err && $err ne $tag && $err ne "$tag\n";
  my $opresultxml;
  eval {
    $opresultxml = BSUtil::toxml($opresult, $BSXML::opstatus);
  };  
  $opresultxml ||= "<status code=\"500\"\n  <summary>internal error in errreply</summary>\n</status>\n";
  BSServer::reply($opresultxml, "Status: $code $tag", 'Content-Type: text/xml', @hdrs);
}

sub getkiwitreefiles {
  my ($cgi, $jobid) = @_;

  my $state = getstate($jobid, 1);
  my $kiwitreefile = "$buildroot/.build.packages/.kiwitree";
  die("kiwitreefile does not exist\n") unless -e $kiwitreefile;
  die("kiwitreefile does not work in rpmhdrs_only mode\n") if -e "$kiwitreefile.rpmhdrs_only";
  my @send;
  my %ok;
  my $kiwidir;
  if (-d "$buildroot/.build.packages/KIWI") {
    $kiwidir = "$buildroot/.build.packages/KIWI";
  } elsif (-d "$buildroot/.build.packages/PRODUCT") {
    $kiwidir = "$buildroot/.build.packages/PRODUCT";
  }
  die("could not determine product build directory\n") unless $kiwidir;
  for my $f (@{$cgi->{'file'} || []}) {
    die("bad file $f\n") if "/$f/" =~ m!//!s;           # no dup slashes
    die("bad file $f\n") if "/$f/" =~ m!/\.\.?/!s;      # no . or ..
    my @f = split('/', $f);
    die("bad file $f\n") unless @f;
    my $c = pop(@f);
    $f = '';
    for (@f) {
      $f .= $_;
      die("bad file $f\n") if !$ok{$f} && (-l "$kiwidir/$f" || ! -d _);
      $ok{$f} = 1;
      $f .= '/';
    }
    $f .= $c;
    die("bad file $f\n") if -l "$kiwidir/$f" || ! -f _;
    push @send, { 'name' => $f, 'filename' => "$kiwidir/$f" };
  }
  BSServer::reply_cpio(\@send);
  return undef;
}

sub jobinfo {
  my ($cgi, $jobid) = @_;
  my $state = getstate($jobid);
  my $info;
  if ($state->{'state'} eq 'building') {
    $info = readstr("$statedir/job");
  } else {
    $info = "<buildinfo>\n  <error>".$state->{'state'}."</error>\n</buildinfo>\n";
  }
  return $info;
}

sub workerinfo {
  my ($cgi, $jobid) = @_;
  my $state = getstate($jobid);
  my $info = {'hostarch' => $hostarch, 'port' => $port};
  $info->{'workerid'} = $workerid if defined $workerid;
  # $info->{'job'} = $state->{'job'};
  # $info->{'arch'} = $state->{'arch'};
  return ($info, $BSXML::worker);
}

sub getlogfile {
  my ($cgi, $jobid) = @_;
  my $state = getstate($jobid, 1);
  if ($cgi->{'view'} && $cgi->{'view'} eq 'entry') {
    my @s = stat("$buildroot/.build.log");
    die("$buildroot/.build.log: $!\n") unless @s;
    my $dir = { 'entry' => [ {'name' => '_log', 'size' => $s[7], 'mtime' => $s[9]} ] };
    return ($dir, $BSXML::dir);
  }
  stream_logfile($jobid, $cgi->{'nostream'}, $cgi->{'start'}, $cgi->{'end'});
  return undef;
}

sub killjob {
  my ($cgi, $jobid) = @_;
  changestate($jobid, 'killed');
  my $state = lockstate();
  if ($state->{'state'} eq 'killed') {
    die("could not kill job\n") unless kill_job();
    BSUtil::appendstr("$buildroot/.build.log", "\n\nKilled job\n");
  }
  unlockstate();
  return "<status=\"ok\" />\n";
}

sub discardjob {
  my ($cgi, $jobid) = @_;
  changestate($jobid, 'discarded');
  my $state = lockstate();
  if ($state->{'state'} eq 'discarded') {
    die("could not kill job\n") unless kill_job();
    BSUtil::appendstr("$buildroot/.build.log", "\n\nDiscarded job\n");
  }
  unlockstate();
  return "<status=\"ok\" />\n";
}

sub badhostjob {
  my ($cgi, $jobid) = @_;
  changestate($jobid, 'badhost');
  my $state = lockstate();
  if ($state->{'state'} eq 'badhost') {
    die("could not kill job\n") unless kill_job();
    BSUtil::appendstr("$buildroot/.build.log", "\n\nTriggered badhost state for job\n");
  }
  unlockstate();
  return "<status=\"ok\" />\n";
}

sub sysrqjob {
  my ($cgi, $jobid) = @_;
  my $state = getstate($jobid, 1);
  die("could not send sysrq\n") unless sysrq_job($cgi->{'sysrq'});
  BSUtil::appendstr("$buildroot/.build.log", "\n\nSent sysrq $cgi->{'sysrq'} to job\n");
  return "<status=\"ok\" />\n";
}

sub startbuild {
  my ($cgi, $jobid) = @_;

  if ($vm eq 'xen' && $xenstore_maxsize && 0 + (-s '/var/lib/xenstored/tdb') > $xenstore_maxsize) {
    die("xenstore too big:".(-s '/var/lib/xenstored/tdb')."\n");
  }
  unlink("job.new.$$");
  BSServer::read_file("job.new.$$");
  my $state = lockstate();
  die("I am not idle!\n") unless $state->{'state'} eq 'idle';
  my $req = $BSServer::request;
  if ($cgi->{'workercode'} && $cgi->{'port'} && $cgi->{'workercode'} ne $workercode && !$noworkercheck) {
    unlink("job.new.$$");
    my $peer = $cgi->{'registerserver'} || "http://$req->{'peer'}:$cgi->{'port'}";
    $workercode = getcode('worker', "$peer/getworkercode");
    die("could not update worker code\n") unless $workercode;
    $state->{'state'} = 'rebooting';
    print "activating new worker code $workercode\n";
    commitstate($state);
    hardstatus('rebooting');
    die("rebooting...\n");
  }
  my $infoxml = readstr("job.new.$$");
  die("bad job xml data\n") unless $infoxml =~ /<.*?>/s;
  my $buildinfo = XMLin($BSXML::buildinfo, $infoxml);
  $jobid ||= Digest::MD5::md5_hex($infoxml);
  undef $infoxml;
  $buildinfo->{'jobid'} = $jobid;
  # old buildinfos missed some entries
  if (@{$buildinfo->{'path'} || []}) {
    $buildinfo->{'project'} ||= $buildinfo->{'path'}->[0]->{'project'};
    $buildinfo->{'repository'} ||= $buildinfo->{'path'}->[0]->{'repository'};
    $buildinfo->{'reposerver'} ||= $buildinfo->{'path'}->[0]->{'server'};
  }
  if ($localkiwi) {
    # make sure this is the right job for us
    my $imagemode = getimagemode($buildinfo);
    my $jobname = ($buildinfo->{'arch'} || '<unknown>') . '/' . ($buildinfo->{'job'} || '<unknown>');
    die("bad job: $jobname: not a product job\n") unless $imagemode && ($imagemode eq 'kiwiproduct' || $imagemode eq 'productcompose');
  }

  $buildcode = codemd5('build');
  if (!$nobuildcodecheck && $cgi->{'buildcode'} && $cgi->{'port'} && $cgi->{'buildcode'} ne $buildcode) {
    print "fetching new buildcode $cgi->{'buildcode'}, mine was $buildcode\n";
    my $peer = $cgi->{'registerserver'} || "http://$req->{'peer'}:$cgi->{'port'}";
    $buildcode = getcode('build', "$peer/getbuildcode");
    die("could not update build code\n") unless $buildcode;
  }
  rename("job.new.$$", 'job') || die("rename job.new.$$ job: $!\n");
  if ($hostcheck) {
    my $server = $buildinfo->{'srcserver'} || $srcserver;
    if (system($hostcheck, '--srcserver', $server, "$statedir/job", 'precheck', $buildroot)) {
      my $res = ($? >> 8) || 1;
      unlink('job');
      die("400 cannot build anything\n") if $res == 3;
      die("400 cannot build this repository\n") if $res == 2;
      die("400 cannot build this package\n");
    }
  }

  $buildinfo->{'nobadhost'} = $cgi->{'nobadhost'} if $cgi->{'nobadhost'};

  # make the BSServer::server() call return
  $BSServer::request->{'returnfromserver'} = [$state, $buildinfo, $cgi->{'registerserver'}];

  return ("<status code=\"failed\">\n  <details>testmode activated</details>\n</status>\n", 'Status: 400 Testmode', 'Content-Type: text/xml') if $testmode;
  return "<status code=\"ok\">\n  <details>so much work, so little time...</details>\n</status>\n";
}

my $idlecnt = 0;
my $rekillcnt = 0;
$oneshot = time() + $oneshot if defined $oneshot;

sub periodic {
  my $state = readxml("$statedir/state", $BSXML::workerstate, 1);
  return unless $state;

  if ($state->{'state'} eq 'idle') {
    if ($oneshot && time() > $oneshot) {
      send_state('exit', $port, $hostarch);
      print "exiting.\n";
      exit(0);
    }
    $idlecnt++;
    if ($idlecnt % 30 == 0) {
      # send idle message every 5 minutes in case the server was down
      $idlecnt = 0;
      send_state('idle', $port, $hostarch) if $state->{'state'} eq 'idle';
    }
  } else {
    $idlecnt = 0;
  }

  if ($state->{'state'} eq 'exit' || $state->{'state'} eq 'dead') {
    $state = lockstate();
    if ($state->{'state'} eq 'dead') {
      close RUNLOCK;
      unlockstate();
      send_state('exit', $port, $hostarch);
      exit(0);
    }
    if ($state->{'state'} eq 'exit') {
      close RUNLOCK;
      $state = {'state' => 'idle'};
      commitstate($state);
      send_state('exit', $port, $hostarch);
      exit(0);
    }
    unlockstate();
  }

  if ($state->{'state'} eq 'rebooting') {
    chdir("$statedir/worker") || die("$statedir/worker: $!");
    close RUNLOCK;
    exec("./bs_worker", @saveargv);
    die("$statedir/worker/bs_worker: $!\n");    # oops
  }

  if ($state->{'state'} eq 'killed' || $state->{'state'} eq 'discarded' || $state->{'state'} eq 'badhost') {
    $rekillcnt++;
    if ($state->{'state'} eq 'discarded' && $state->{'nextstate'}) {
      # signal early that we're going down
      send_state('exit', $port, $hostarch);
      $rekillcnt = 12;
    }
    if ($rekillcnt % 12 == 0) {
      # re-kill after 2 minutes, maybe build is stuck somewhere
      $rekillcnt = 0;
      $state = lockstate();
      if ($state->{'state'} eq 'killed' || $state->{'state'} eq 'discarded' || $state->{'state'} eq 'badhost') {
        kill_job();
      }
      unlockstate();
    }
  } else {
    $rekillcnt = 0;
  }

  if ($state->{'state'} eq 'killed' || $state->{'state'} eq 'discarded' || $state->{'state'} eq 'building' || $state->{'state'} eq 'badhost') {
    if ($state->{'pid'} && !kill(0, $state->{'pid'})) {
      $state = lockstate();
      if ($state->{'state'} eq 'killed' || $state->{'state'} eq 'discarded' || $state->{'state'} eq 'building' || $state->{'state'} eq 'badhost') {
        if ($state->{'pid'} && !kill(0, $state->{'pid'})) {
          # our worker process is gone, should never happen...
          kill_job();   # just in case...
          $state = {'state' => $state->{'nextstate'} || 'idle'};
          commitstate($state);
          next;
        }
      }
      unlockstate();
    }
  }

  if ($state->{'state'} eq 'building') {
    my $locked = -1;
    my $maxsize = $state->{'logsizelimit'} || $buildlog_maxsize;
    my $maxidle = $state->{'logidlelimit'} || $buildlog_maxidle;
    while ($locked++ < 1) {
      $state = lockstate() if $locked == 1;
      last if $state->{'state'} ne 'building';
      my $ct = time();
      my @s = stat("$buildroot/.build.log");
      next unless @s;
      if ($maxsize && $s[7] > $maxsize) {
	next unless $locked;
	if (!kill_job()) {
	  warn("could not kill job\n");
	  last;
	}
	trunc_logfile("$buildroot/.build.log");
	$state->{'state'} = 'killed';
	commitstate($state);
	$locked = 0;
      } elsif ($maxidle && $ct - $s[9] > $maxidle) {
	next unless $locked;
	if (!kill_job()) {
	  warn("could not kill job\n");
	  last;
	}
	append_nonfatal("$buildroot/.build.log", "\n\n$maxidle seconds (logidlelimit) elapsed without any output: build job terminated!\n");
	$state->{'state'} = 'killed';
	commitstate($state);
	$locked = 0;
      } elsif ($s[7] > 250) {
	my $tail = tail_logfile("$buildroot/.build.log", 250) || '';
	my $kill_job;
	$kill_job = "kvm memory page bug" if $tail =~ /BUG: unable to handle kernel NULL pointer dereference at/;
	$kill_job = "kvm spinnlock bug" if $tail =~ /INFO: rcu_sched self-detected stall on CPU/;
# leads to false positive on i586 kernels
#	$kill_job = "xen soft lockup" if $tail =~ /BUG: soft lockup - CPU#\d+ stuck for/;
	$kill_job = "kvm soft lockup" if $tail =~ /BUG: soft lockup - CPU#\d+ stuck for (\d+)s/ && $1 > 1800;
	$kill_job = "stuck in SLOF" if $tail =~ /\( \d+ \) Data Storage Exception/;
	if ($kill_job) {
	  next unless $locked;
	  print "detected $kill_job, discarding job\n";
	  if (!kill_job()) {
	    warn("could not kill job\n");
	    last;
	  }
	  append_nonfatal("$buildroot/.build.log", "\n\nDetected $kill_job, retrying on another host\n");
	  $state->{'state'} = 'badhost';
	  commitstate($state);
	  $locked = 0;
	}
      }
      last;
    }
    unlockstate() if $locked;
  }
}

return 1 if $testcasemode;

# better safe than sorry...
my $startworkercode = '?';
$startworkercode = codemd5('.') if -e './bs_worker';
chdir($statedir) || die("$statedir: $!\n");

BSServer::deamonize(@ARGV) unless $oneshot;
$SIG{'PIPE'} = 'IGNORE';
$| = 1;

die("Unsupported proto $proto\n") if $proto && $proto ne 'http' && $proto ne 'https';
require BSSSL if $proto && $proto eq 'https';	# fail early

# calculate code meta md5
$workercode = codemd5('worker');
$buildcode = codemd5('build');
$workercode = $buildcode = '?' if $startworkercode ne $workercode;

open(RUNLOCK, '>>', "$statedir/lock") || die("$statedir/lock: $!");
flock(RUNLOCK, LOCK_EX | LOCK_NB) || die("worker is already running on $statedir!\n");
utime undef, undef, "$statedir/lock";

# we always start idle
lockstate();
unlink("$statedir/job");
unlink("$buildroot/.build.log");
commitstate({'state' => 'idle'});

# start server process...
BSServer::serveropen($port ? $port : \$port, undef, undef, undef, $bindaddress);

print "worker started on port $port code $workercode build $buildcode\n";
mkdir($buildroot) unless -d $buildroot;
send_state('idle', $port, $hostarch);
hardstatus('idle');

my $dispatches = [
  '/kiwitree $jobid:? $file:*' => \&getkiwitreefiles,
  '/info $jobid:?' => \&jobinfo,
  '/worker $jobid:?' => \&workerinfo,
  '/logfile $jobid:? nostream:? start:? end:? view:?' => \&getlogfile,
  '/kill $jobid:?' => \&killjob,
  '/discard $jobid:?' => \&discardjob,
  '/badhost $jobid:?' => \&badhostjob,
  '/sysrq $jobid:? sysrq:?' => \&sysrqjob,
  'PUT:/build $jobid:? buildcode:? workercode:? port:? registerserver:? nobadhost:? *:?' => \&startbuild,
];

my $conf = {
  'dispatches' => $dispatches,
  'periodic' => \&periodic,
  'periodic_interval' => 10,
  'accept' => sub {close(RUNLOCK)},
};


BSDispatch::compile($conf);
$conf->{'stdreply'} ||= \&stdreply;
$conf->{'errorreply'} ||= \&errreply;
$conf->{'dispatch'} ||= \&BSDispatch::dispatch;
if ($proto && $proto eq 'https') {
  $conf->{'proto'} = $proto;
  $conf->{'ssl_keyfile'} = $BSConfig::ssl_keyfile if $BSConfig::ssl_keyfile;
  $conf->{'ssl_certfile'} = $BSConfig::ssl_certfile if $BSConfig::ssl_certfile;
}

# run the server, this will return only for /build requests
# see returnfromserver setting above
my ($state, $buildinfo, $registerserver) = BSServer::server($conf);
# the returned state is locked

print "got job, run build...\n";
BSServer::done(1);

if (!$testmode && $buildinfo->{'masterdispatched'}) {
  $buildinfo->{'jobid'} = $buildinfo->{'masterdispatched'};
  eval { send_dispatched($buildinfo->{'reposerver'}, $port, $hostarch, $buildinfo) };
  if ($@) {
    unlink('job');
    die($@);
  }
}

mkdir_p($buildroot) unless -e $buildroot;
unlink("$buildroot/.build.meta");
unlink("$buildroot/.build.packages");
unlink("$buildroot/.build.log");
writestr("$buildroot/.build.log", undef, '');

$state->{'state'} = 'building';
$state->{'jobid'} = $buildinfo->{'jobid'};
$state->{'pid'} = $$;
$state->{'logidlelimit'} = $buildinfo->{'logidlelimit'} if $buildinfo->{'logidlelimit'};
$state->{'logsizelimit'} = $buildinfo->{'logsizelimit'} if $buildinfo->{'logsizelimit'};
commitstate($state);

hardstatus("$buildinfo->{'arch'} $buildinfo->{'package'}");
send_state('building', $port, $hostarch, $registerserver || $buildinfo->{'reposerver'});

my $ex;
eval {
  $ex = dobuild($buildinfo);
};
if ($@) {
  $ex = 1;
  $ex = 3 if !$buildinfo->{'download_complete'} && $@ !~ /^\d\d\d/;
  # mark as bad build host if Build.pm could not be imported
  $ex = 3 unless defined &Build::queryhdrmd5;
  $ex = 3 if $@ =~ /^500 /;
  # catch disk full errors before the build script could be even started
  $ex = 3 if $@ =~ /No space left on device/;
  $ex = 4 if $@ =~ /^FATAL:/;
  append_nonfatal("$buildroot/.build.log", $workerid ? "$@(worker was $workerid)\n" : $@);
  print $@;
}

if ($buildinfo->{'followupfile'}) {
  my $buildsrcdir = "$buildroot/.build.packages/SOURCES";
  $buildsrcdir = "$buildroot/.build-srcdir" if $vm =~ /(xen|kvm|qemu|emulator)/;
  # if it was a follow up build, prepend old logfile
  if (-s "$buildsrcdir/logfile") {
    local *F;
    local *T;
    if (open(F, '<', "$buildroot/.build.log")) {
      if (open(T, '>>', "$buildsrcdir/logfile")) {
	my $buf;
	syswrite(T, $buf) while sysread(F, $buf, 8192);
	close T;
	rename("$buildsrcdir/logfile", "$buildroot/.build.log");
      }
      close F;
    }
  }
  if (-d "$buildroot/.build.packages/OTHER") {
    for my $f ('rpmlint.log', '_statistics', '_buildenv') {
      next unless -s "$buildsrcdir/$f";
      BSUtil::cp("$buildsrcdir/$f", "$buildroot/.build.packages/OTHER/$f");
    }
    if (-e "$buildsrcdir/_ccache.tar") {
      link_or_copy("$buildsrcdir/_ccache.tar", "$buildroot/.build.packages/OTHER/_ccache.tar") || die("_ccache.tar link_or_copy failed")
    }
  }
  # delete follow-up srcrpm
  for my $srcrpm (ls("$buildroot/.build.packages/SRPMS")) {
    unlink("$buildroot/.build.packages/SRPMS/$srcrpm") unless -e "$buildsrcdir/$srcrpm";
  }
}

# build is done, send back result
$state = lockstate();

if ($state->{'state'} eq 'discarded') {
  # our poor job is no longer needed
  print "build discarded...\n";
  unlink("$buildroot/.build.log");
  unlink("$buildroot/job");
  cleanup_job();

  if ($state->{'nextstate'}) {
    $state = {'state' => $state->{'nextstate'}};
    commitstate($state);
    exit(0);
  }
  $state = {'state' => 'idle'};
  commitstate($state);
  hardstatus('idle');
  exit(0) if $oneshot && time() > $oneshot;
  send_state('idle', $port, $hostarch);
  exit(0);
}

if ($state->{'state'} eq 'badhost') {
  $ex = 3;
} elsif ($state->{'state'} ne 'building') {
  $ex = 1;	# something is wrong, consider job bad
}

if (! -s "$buildroot/.build.log") {
  eval {
    if (defined($workerid)) {
      writestr("$buildroot/.build.log", undef, "build on $workerid did not create a logfile\n");
    } else {
      writestr("$buildroot/.build.log", undef, "build did not create a logfile\n");
    }
  };
  $ex = 3;
}

if ($hostcheck) {
  print "running post-build host check\n";
  my $server = $buildinfo->{'srcserver'} || $srcserver;
  if (system($hostcheck, '--srcserver', $server, "$statedir/job", $ex ? 'failed' : 'succeeded', "$buildroot/.build.log")) {
    print "post-build host check failed\n";
    $ex = 3;
  }
}

my $send_genbuildreqs;
if ($ex != 3 && $ex != 4 && -e "$buildroot/.build.packages/OTHER/_generated_buildreqs") {
  if (-s "$buildroot/.build.packages/OTHER/_generated_buildreqs" > 100000) {
    $ex = 1;
  } else {
    my $genbuildreqs = readstr("$buildroot/.build.packages/OTHER/_generated_buildreqs");
    $send_genbuildreqs = { name => '_generated_buildreqs', filename => "$buildroot/.build.packages/OTHER/_generated_buildreqs" };
    if ($genbuildreqs =~ /[^\n\040-\176]/s) {
      $ex = 1;
      $send_genbuildreqs = undef;
    } elsif (Digest::MD5::md5_hex($genbuildreqs) ne ($buildinfo->{'genbuildreqs'} || '')) {
      $ex = 11;
    }
  }
}

# collect files we want to send
my @send;
my $kiwitree;
if ($ex == 0) {
  my @d;
  push @d, map {"RPMS/$_"} sort(ls("$buildroot/.build.packages/RPMS"));
  push @d, 'SRPMS';
  @d = ('DEBS') if $buildinfo->{'file'} =~ /(?:\.dsc|build\.collax|debian\.control)$/;
  if (-d "$buildroot/.build.packages/SDEBS") {
    @d = map {"DEBS/$_"} sort(ls("$buildroot/.build.packages/DEBS"));	# assume debbuild
    push @d, 'SDEBS';
  }
  @d = ('ARCHPKGS') if $buildinfo->{'file'} =~ /PKGBUILD$/;
  @d = ('APKS') if $buildinfo->{'file'} =~ /APKBUILD$/;
  @d = ('KIWI') if $buildinfo->{'file'} =~ /\.kiwi$/;
  @d = ('PRODUCT') if $buildinfo->{'file'} =~ /\.productcompose$/;
  @d = ('DOCKER') if $buildinfo->{'file'} =~ /Dockerfile(?:\.|$)/;
  @d = ('FISSILE') if $buildinfo->{'file'} =~ /fissile\.yml$/;
  @d = ('HELM') if $buildinfo->{'file'} =~ /Chart\.yaml$/;
  push @d, 'OTHER';
  for my $d ('.', @d) {
    my @files = sort(ls("$buildroot/.build.packages/$d"));
    @files = grep {$_ ne 'same_result_marker' && $_ ne '.kiwitree'} @files;
    if ($d eq 'KIWI' || $d eq 'PRODUCT') {
      $kiwitree = [ grep {-d "$buildroot/.build.packages/$d/$_"} @files ];
      undef $kiwitree unless @$kiwitree;
      undef $kiwitree if defined($BSConfig::nokiwitree) && $BSConfig::nokiwitree;
    }
    @files = grep {! -l "$buildroot/.build.packages/$d/$_" && -f _} @files;
    # normalize/compress containers if we have /usr/bin/obs-gzip-go
    if ($can_rewrite_containers && -x "/usr/bin/obs-gzip-go") {
      for my $containerinfo (grep {/\.containerinfo$/} @files) {
	my $container = "$buildroot/.build.packages/$d/$containerinfo";
        $container =~ s/\.containerinfo$/\.tar/;
        next unless ! -l $container && -f _;
        eval {
          normalize_container($container, 1, "$buildroot/.build.packages");
          BSUtil::touch("$container.recompressed");
	  push @send, "$container.recompressed" if ! -l "$container.recompressed" && -f _;
        };
        warn($@) if $@;
      }
    }
    push @send, map {"$buildroot/.build.packages/$d/$_"} @files;
  }
  @send = map {{name => (split('/', $_))[-1], filename => $_}} @send;
  @send = grep {$_->{'name'} ne '_generated_buildreqs'} @send;

  # add slsa provenance files if requested
  @send = grep {$_->{'name'} !~ /slsa_provenance.(?:json|config)$/} @send;
  if ($buildinfo->{'slsaprovenance'}) {
    my $slsa_provenance;
    if (($buildinfo->{'slsa_provenance_version'} || '') eq 'v1') {
      $slsa_provenance = generate_slsa_provenance_statement_v1($buildinfo, \@send);
    } else {
      $slsa_provenance = generate_slsa_provenance_statement_v02($buildinfo, \@send);
    }
    push @send, { 'name' => '_slsa_provenance.json', 'data' => $slsa_provenance };
    push @send, { 'name' => '_slsa_provenance.config', 'data' => $buildinfo->{'slsa_provenance_config'}} if $buildinfo->{'slsa_provenance_config'};
  }
  
  if ($kiwitree) {
    my $kiwitreefile = "$buildroot/.build.packages/.kiwitree";
    eval {
      die(".kiwitree already exists\n") if -e $kiwitreefile;
      my $kiwidir = "$buildroot/.build.packages/KIWI";
      $kiwidir = "$buildroot/.build.packages/PRODUCT" if $buildinfo->{'file'} =~ /\.productcompose$/;
      buildkiwitree($kiwitreefile, map { ("$kiwidir/$_", "$_/") } @$kiwitree);
      BSUtil::touch("$kiwitreefile.rpmhdrs_only") if $buildinfo->{'rpmhdrs_only'};
      push @send, { name => '.kiwitree', filename => $kiwitreefile };
    };
    if ($@) {
      print "could not create kiwitree: $@\n";
      BSUtil::appendstr("$buildroot/.build.log", "\ncould not create kiwitree: $@\n");
      $ex = 1;
    }
  }
  if (!@send) {
    print "build did not create anything to send back!\n";
    $ex = 1;
  }
} elsif ($ex == 2) {
  my $statsfile = "$buildroot/.build.packages/OTHER/_statistics";
  push @send, { name => '_statistics', filename => $statsfile } if -e $statsfile;
  # ccache is enabled and there is no tar from previous builds,
  # this is a good ccache archive and should be used for subsequent builds.
  my $ccachetar = "$buildroot/.build.packages/OTHER/_ccache.tar";
  my $occachetar = "$buildroot/.build.oldpackages/_ccache.tar";
  push @send, {name => '_ccache.tar', filename => $ccachetar} if -e $ccachetar && !-e $occachetar;
}

my $code;
if (!$ex) {
  print "build succeeded, sending everything back...\n";
  $code = 'succeeded';
} elsif ($ex == 2) {
  print "build succeeded, but does not differ from old build result...\n";
  $code = 'unchanged';
} elsif ($ex == 3 && $buildinfo->{'nobadhost'}) {
  print "build failed (ignored bad build host flag)...\n";
  BSUtil::appendstr("$buildroot/.build.log", "\n$buildinfo->{'nobadhost'}\n");
  $code = 'failed';
} elsif ($ex == 3) {
  print "build failed, marked as bad build host...\n";
  $code = 'badhost';
  # Check if buildconfig is just broken and won't ever succeed
  my $data = tail_logfile("$buildroot/.build.log", 10240);
  if ($data) {
    # Remove timing prefix, e.g.
    # [   41s]
    $data =~ s/^(?:\[[\d\s]+s\]\s*)?//gm;
    # Match any of those:
    # run-init: /.build/build: Exec format error
    # /bin/rpm: Exec format error
    # /bin/sh: error while loading shared libraries: libgcc_s.so.1: cannot open shared object file
    if ($data =~ /^(\S{1,30}): error while loading shared libraries: lib/m
        || $data =~ /^(?:run-init: )?\S{1,3330}: Exec format error/m) {
      print "Wait ... spotted failed build. Host is good.\n";
      $code = 'failed';
    }
  }
} elsif ($ex == 4) {
  print "detected broken build host, exiting...\n";
  $code = 'badhost';
  $state->{'nextstate'} = 'dead';
} elsif ($ex == 11) {
  print "build has different generated build requires\n";
  $code = 'genbuildreqs';
} else {
  print "build failed, send back logfile...\n";
  $code = 'failed';
}

# always send the log and the meta files
push @send, {name => 'meta', filename => "$buildroot/.build.meta"} if -e "$buildroot/.build.meta";
push @send, {name => 'logfile', filename => "$buildroot/.build.log"};

if ($code eq 'badhost') {
  # only transmit the logfile in the badhost case
  @send = ();
  if (-s "$buildroot/.build.log") {
    push @send, {name => 'logfile', filename => "$buildroot/.build.log"};
  } elsif (-e "$buildroot/.build.log") {
    push @send, {name => 'logfile', data => "empty build log?\n"};
  } else {
    push @send, {name => 'logfile', data => "no build log?\n"};
  }
}
push @send, $send_genbuildreqs if $send_genbuildreqs && $ex != 2 && $ex != 3 && $ex != 4;

if (!$testmode) {
  my $param = {
    uri => "$buildinfo->{'reposerver'}/putjob",
    request => 'POST',
    headers => [ 'Content-Type: application/x-cpio' ],
    chunked => 1,
    data => \&BSHTTP::cpio_sender,
    cpiofiles => \@send,
  };
  my $now = time();
  my @args = ("job=$buildinfo->{'job'}", "arch=$buildinfo->{'arch'}", "jobid=$buildinfo->{'jobid'}", "code=$code", "now=$now");
  push @args, "workerid=$workerid" if defined $workerid;
  push @args, "kiwitree=1" if $kiwitree && $code eq 'succeeded';
  eval {
    my $res = BSRPC::rpc($param, undef, @args);
  };
  if ($@) {
    print "rpc failed: $@\nsleeping one minute just in case...\n";
    sleep(60);
  } else {
    print "sent, all done...\n";
    # do the wipe
    wipe_all() if $wipeafterbuild;
  }
} else {
  print "testmode, not sending anything\n";
  print Dumper(\@send);
  BSUtil::appendstr('testmode.log', BSUtil::isotime(time).": $buildinfo->{'arch'}/$buildinfo->{'job'}: $code\n");
}

unlink("$buildroot/.build.log");
unlink("$buildroot/job");
print "\n";
cleanup_job();

if ($state->{'nextstate'}) {
  $state = {'state' => $state->{'nextstate'}};
  commitstate($state);
  exit(0);
}
$state = {'state' => 'idle'};
commitstate($state);
hardstatus('idle');

exit(0) if $oneshot && time() > $oneshot;
send_state('idle', $port, $hostarch);

exit(0);
