#!/usr/bin/python

from __future__ import annotations

import os
import sys
import traceback
import logging
import time
from collections import defaultdict
from typing import List, Any, DefaultDict, Tuple, Optional, cast
from collections.abc import Set
from pathlib import Path
import graphlib
import datetime
import threading
from concurrent.futures import (
  ThreadPoolExecutor, Future,
  wait as futures_wait, FIRST_COMPLETED,
)
import subprocess
from functools import partial
import json

import prctl
import structlog
from structlog.types import Processor

topdir = Path(__file__).resolve().parent

from lilac2.vendor.myutils import lock_file
from lilac2.vendor.serializer import PickledData
from lilac2.vendor.nicelogger import enable_pretty_logging

from lilac2.packages import (
  DependencyManager, get_dependency_map, get_changed_packages,
  Dependency,
)
from lilac2.cmd import (
  run_cmd, git_pull_override, git_push, pkgrel_changed,
  git_reset_hard, get_git_branch,
)
from lilac2 import tools
from lilac2.repo import Repo
from lilac2.const import mydir, _G
from lilac2.nvchecker import packages_need_update, nvtake, NvResults
from lilac2.nomypy import BuildResult, BuildReason # type: ignore
from lilac2.building import build_package, MissingDependencies
from lilac2 import slogconf
from lilac2 import intl
from lilac2.workerman import WorkerManager
from lilac2.typing import PkgToBuild, Rusages
try:
  from lilac2 import db
except ImportError:
  class db: # type: ignore
    USE = False

config = tools.read_config()
TLS = threading.local()

# Setting up environment variables
os.environ.update(config.get('envvars', ()))
os.environ['PATH'] = str(topdir) + ':' + os.environ['PATH']

DESTDIR = Path(config['repository'].get('destdir', '')).expanduser()
MYNAME = config['lilac']['name']

nvdata: dict[str, NvResults] = {}
DEPMAP: dict[str, set[Dependency]] = {}
BUILD_DEPMAP: dict[str, set[Dependency]] = {}
build_reasons: DefaultDict[str, list[BuildReason]] = defaultdict(list)

logger = logging.getLogger(__name__)
build_logger_old = logging.getLogger('build')
build_logger = structlog.get_logger(logger_name='build')
REPO = _G.repo = Repo(config)
_G.reponame = REPO.name

EMPTY_COMMIT = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'

def setup_build_logger() -> None:
  handler = logging.FileHandler(mydir / 'build.log')
  handler.setFormatter(logging.Formatter('[%(asctime)s] %(message)s', '%Y-%m-%d %H:%M:%S'))
  build_logger_old.addHandler(handler)

  logfile = (mydir / 'build-log.json').open('a')

  processors: List[Processor] = [
    slogconf.exc_info,
    slogconf.add_timestamp,
    structlog.processors.format_exc_info,
    slogconf.json_renderer,
  ]

  logger_factory = structlog.PrintLoggerFactory(file=logfile)

  structlog.configure(
    processors = processors,
    logger_factory = logger_factory,
  )

def git_last_commit() -> str:
  cmd = ['git', 'log', '-1', '--format=%H']
  return run_cmd(cmd).strip()

def packages_with_depends(
  repo: Repo,
) -> Tuple[graphlib.TopologicalSorter, dict[str, set[str]]]:
  dep_building_map: dict[str, set[str]] = {}
  nonexistent: DefaultDict[str, List[Dependency]] = defaultdict(list)
  for name in tuple(build_reasons):
    ds = DEPMAP[name]
    for d in ds:
      if not d.resolve():
        if not repo.manages(d):
          logger.warning('%s depends on %s, but it\'s not managed.',
                         name, d)
          nonexistent[name].append(d)
          continue

        # don't rebuild a failed dep
        if db.USE and db.is_last_build_failed(d.pkgname):
          continue

        logger.info('build %s as a dependency of %s because no built package found',
                    d.pkgname, name)
        build_reasons[d.pkgname].append(BuildReason.Depended(name))

    dep_building_map[name] = set()
    for x in ds:
      pkgbase = x.pkgdir.name
      dep_building_map[name].add(pkgbase)
      # a dependency may depend other packages, we need their relations
      if pkgbase in DEPMAP:
        dep_building_map[pkgbase] = {
          x.pkgdir.name for x in DEPMAP[pkgbase]}

  l10n = intl.get_l10n('mail')
  for name, deps in nonexistent.items():
    repo.send_error_report(
      repo.lilacinfos[name],
      subject = l10n.format_value(
        'nonexistent-deps-subject', {'pkg': name}),
      msg = l10n.format_value(
        'nonexistent-deps-body',
        {'pkg': name, 'deps': repr(deps), 'count': len(deps)}),
    )

  sorter = graphlib.TopologicalSorter(dep_building_map)

  packages = graphlib.TopologicalSorter(dep_building_map).static_order()
  # filter out already built packages
  packages = [x for x in packages if x in build_reasons]
  for p in packages:
    logger.info('building %s because of %r', p, build_reasons[p])

  if db.USE:
    with db.get_session() as s:
      s.execute('delete from pkgcurrent')

      rows = []
      for idx, pkg in enumerate(packages):
        rs = json.dumps([r.to_dict() for r in build_reasons[pkg]])
        rows.append((pkg, idx, 'pending', rs))
      s.executemany(
        '''insert into pkgcurrent
           (pkgbase, index, status, build_reasons) values
           (%s, %s, %s, %s)''', rows)
      db.build_updated(s)

  return sorter, dep_building_map

def building_priority(revdepmap: dict[str, set[str]], pkg: str) -> int:
  new = {pkg}
  while new:
    depees: set[str] = set()
    new_new = set()
    for p in new:
      if d := revdepmap.get(p):
        new_new.update(d - depees)
        depees.update(d)
    new = new_new
  rs = build_reasons[pkg]
  for p in depees:
    rs.extend(build_reasons[p])
  return min(buildreason_priority(r, revdepmap) for r in rs)

def buildreason_priority(r: BuildReason, revdepmap: dict[str, set[str]]) -> int:
  if isinstance(r, BuildReason.UpdatedPkgrel):
    return 0

  if isinstance(r, BuildReason.NvChecker):
    if any(source == 'manual' for _, source in r.items):
      return 0
    if len(r.items) > 1 or r.items[0][0] > 0:
      # rebuild seen by nvchecker
      return 1

  if isinstance(r, BuildReason.Depended):
    return building_priority(revdepmap, r.depender)

  if isinstance(r, BuildReason.UpdatedFailed):
    return 2

  return 3

class BuildSorter:
  def __init__(
    self,
    sorter: graphlib.TopologicalSorter,
    depmap: dict[str, set[str]],
  ) -> None:
    sorter.prepare()
    self.sorter = sorter
    self.ready: list[str] = []

    revdepmap = defaultdict(set)
    for p, deps in depmap.items():
      for a in deps:
        revdepmap[a].add(p)

    self.priority_func = partial(building_priority, revdepmap)

  def is_active(self) -> bool:
    return self.sorter.is_active()

  def done(self, pkg: str) -> None:
    try:
      self.ready.remove(pkg)
      self.sorter.done(pkg)
    except ValueError:
      # we may try to remove a pkg twice because we may run check_buildability
      # twice: once for regular round, once for picking while starving
      pass

  def get_ready(self) -> tuple[str, ...]:
    new = self.sorter.get_ready()
    while new:
      self.ready += [x for x in new if x in build_reasons]
      self.sorter.done(*[x for x in new if x not in build_reasons])
      new = self.sorter.get_ready()
    logger.debug('ready-to-build packages: %s', self.ready)
    return tuple(self.ready)

def start_build(
  repo: Repo,
  logdir: Path,
  failed: dict[str, tuple[str, ...]],
  built: set[str],
  workermans: list[WorkerManager],
) -> None:
  # built is used to collect built package names
  sorter, depmap = packages_with_depends(repo)

  max_workers = sum(wm.max_concurrency for wm in workermans)
  try:
    buildsorter = BuildSorter(sorter, depmap)
    futures: dict[Future, PkgToBuild] = {}
    with ThreadPoolExecutor(
      max_workers = max_workers,
      initializer = setup_thread,
    ) as executor:
      # initialize all threads so they get their worker_no
      executor.map(lambda x: None, range(max_workers))

      while True:
        pkgs = try_pick_some(
          repo,
          buildsorter, failed,
          running = frozenset(x.pkgbase for x in futures.values()),
          starving = not bool(futures),
          workermans = workermans,
        )
        for pkg in pkgs:
          if pkg.pkgbase not in nvdata:
            # this can happen when cmdline packages are specified and
            # a package is pulled in by OnBuild
            logger.warning('%s not in nvdata, skipping', pkg.pkgbase)
            buildsorter.done(pkg.pkgbase)
            wm = cast(WorkerManager, pkg.workerman)
            wm.current_task_count -= 1
            continue
          fu = executor.submit(
            build_it, pkg, repo, buildsorter, built, failed)
          futures[fu] = pkg

        if not pkgs and not futures:
          # no more packages and no task is running: we're done
          break

        done, pending = futures_wait(futures, return_when=FIRST_COMPLETED)
        for fu in done:
          pkg = futures.pop(fu)
          wm = cast(WorkerManager, pkg.workerman)
          wm.current_task_count -= 1
          fu.result()

        # at least one task is done, try pick new tasks

  except KeyboardInterrupt:
    logger.info('keyboard interrupted, bye~')

def try_pick_some(
  repo: Repo,
  buildsorter: BuildSorter,
  failed: dict[str, tuple[str, ...]],
  running: Set[str],
  starving: bool,
  workermans: list[WorkerManager],
) -> list[PkgToBuild]:
  if not buildsorter.is_active():
    return []

  ready = buildsorter.get_ready()
  if not ready:
    return []

  ready_to_build = [pkg for pkg in ready if pkg not in running]
  if not ready_to_build:
    return []

  if db.USE:
    rusages = db.get_pkgs_last_rusage(ready_to_build)
  else:
    rusages = Rusages({})

  ret: list[PkgToBuild] = []

  for wm in workermans:
    if not ready_to_build:
      break

    name = wm.name
    ready_to_build_wm = [
      x for x in ready_to_build
      if not (
        (info := repo.lilacinfos.get(x))
        and info.allowed_workers
        and name not in info.allowed_workers
      )
    ]

    if not ready_to_build_wm:
      continue

    to_builds = wm.try_accept_package(
      ready_to_build_wm,
      rusages,
      buildsorter.priority_func,
      lambda pkg: check_buildability(pkg,repo, buildsorter, failed),
    )
    ret.extend(to_builds)
    # remove picked packages from ready_to_build
    picked = {x.pkgbase for x in to_builds}
    ready_to_build = [x for x in ready_to_build
                      if x not in picked]

  if not ret and starving:
    wm = workermans[0]
    def sort_key(pkg):
      p = buildsorter.priority_func(pkg)
      r = rusages.for_package(pkg, [wm.name])
      if r is not None:
        m = r.memory
      else:
        m = 10 * 1024**3
      return (p, m)
    ready_to_build.sort(key=sort_key)
    logger.debug('sorted ready_to_build: %r', ready_to_build)
    memory_avail = wm.get_resource_usage()[1]
    logger.info('insufficient memory, starting only one build on %s (available: %d)', wm.name, memory_avail)
    for pkg in ready_to_build:
      to_build = check_buildability(pkg, repo, buildsorter, failed)
      if to_build is None:
        continue
      to_build.workerman = wm
      ret.append(to_build)
      break

  return ret

def check_buildability(
  pkg: str,
  repo: Repo,
  buildsorter: BuildSorter,
  failed: dict[str, tuple[str, ...]],
) -> Optional[PkgToBuild]:
  '''NOTE: caller needs to set workerman on returned value'''
  to_build = PkgToBuild(pkg)

  if pkg in failed:
    buildsorter.done(pkg)
    if db.USE:
      with db.get_session() as s:
        db.mark_pkg_as(s, pkg, 'done')
        db.build_updated(s)
    # marked as failed (by lilac loader), skip
    return None

  if rs := build_reasons.get(pkg):
    if len(rs) == 1 and isinstance(rs[0], BuildReason.FailedByDeps):
      ds = BUILD_DEPMAP[pkg]
      if not all(d.resolve() for d in ds):
        buildsorter.done(pkg)
        if db.USE:
          with db.get_session() as s:
            db.mark_pkg_as(s, pkg, 'done')
            db.build_updated(s)
        # deps are still missing, skip
        return None
    if on_build := next(
      (r for r in rs if isinstance(rs[0], BuildReason.OnBuild)),
      None
    ):
      update_on_build = on_build.update_on_build
      if len(rs) == 1 and any(
        x.pkgbase in failed for x in update_on_build
      ):
        # on_build packages have failures, skip
        buildsorter.done(pkg)
        if db.USE:
          with db.get_session() as s:
            db.mark_pkg_as(s, pkg, 'done')
            db.build_updated(s)
        return None
      try:
        if db.USE:
          vers = db.get_update_on_build_vers(update_on_build)
        else:
          vers = []
        if len(rs) == 1 and vers and all(old == new for old, new in vers):
          # no need to rebuild
          buildsorter.done(pkg)
          with db.get_session() as s:
            db.mark_pkg_as(s, pkg, 'done')
            db.build_updated(s)
          return None
        logger.debug('on_build_vers: %r', vers)
        to_build = PkgToBuild(pkg, vers)
      except Exception as e:
        logger.exception('get_update_on_build_vers')
        mod = repo.lilacinfos[pkg]
        l10n = intl.get_l10n('mail')
        repo.send_error_report(
          mod,
          subject = l10n.format_value('update_on_build-error'),
          exc = e,
        )
        return None

  if db.USE:
    if mod2 := REPO.lilacinfos.get(pkg):
      update_on_build = mod2.update_on_build
      if update_on_build and not to_build.on_build_vers:
        # Fill in on_build_vers for packages that are not triggered by OnBuild.
        # Provide the last version string as both old and new since it's not
        # triggered by a change of them.
        vers = [
          (new, new) for _, new in db.get_update_on_build_vers(update_on_build)
        ]
        to_build = PkgToBuild(pkg, vers)
    else:
      logger.warning('%s not in lilacinfos.', pkg)

  return to_build

def build_it(
  to_build: PkgToBuild, repo: Repo, buildsorter: BuildSorter,
  built: set[str], failed: dict[str, tuple[str, ...]],
) -> None:
  pkg = to_build.pkgbase
  logger.info('building %s', pkg)
  logfile = logdir / f'{pkg}.log'
  wm = cast(WorkerManager, to_build.workerman)
  worker_no = TLS.worker_no - wm.workers_before_me

  if db.USE:
    with db.get_session() as s:
      db.mark_pkg_as(s, pkg, 'building')
      db.build_updated(s)

  rs = build_reasons.get(pkg)
  commit_msg_template = [
    f'{repo.commit_msg_prefix}{pkg}: auto updated to {{built_version}}',
    '',
    'It has been built because:',
  ]
  if rs:
    commit_msg_template.extend(f'* {r}' for r in rs)
  else:
    commit_msg_template.append('unknown reasons?!')

  r, version = build_package(
    to_build, repo.lilacinfos[pkg],
    update_info = nvdata[pkg],
    commit_msg_template = '\n'.join(commit_msg_template),
    bindmounts = repo.bindmounts,
    tmpfs = repo.tmpfs,
    depends = BUILD_DEPMAP.get(pkg, ()),
    repo = REPO,
    myname = MYNAME,
    destdir = DESTDIR,
    logfile = logfile,
    worker_no = worker_no,
  )

  elapsed = r.elapsed
  logger.info(
    'package %s (version %s) finished in %ds with result: %r',
    pkg, version, elapsed, r,
  )
  repo.on_built(pkg, r, version)

  newver = nvdata[pkg].newver
  msg = None

  if isinstance(r, BuildResult.successful):
    build_logger_old.info(
      '%s %s [%s] successful after %ds',
      pkg, newver, version, elapsed)
    build_logger.info(
      'successful', pkgbase = pkg,
      nv_version = newver, pkg_version = version, elapsed = elapsed,
    )

  elif isinstance(r, BuildResult.staged):
    build_logger_old.info(
      '%s %s [%s] staged after %ds',
      pkg, newver, version, elapsed)
    build_logger.info(
      'staged', pkgbase = pkg,
      nv_version = newver, pkg_version = version, elapsed = elapsed,
    )

  elif isinstance(r, BuildResult.skipped):
    build_logger_old.warning('%s %s skipped after %ds', pkg, newver, elapsed)
    build_logger.warning(
      'skipped', pkgbase = pkg,
      nv_version = newver, msg = r.reason, elapsed = elapsed,
    )
    msg = r.reason

  elif isinstance(r, BuildResult.failed):
    build_logger_old.error('%s %s [%s] failed after %ds', pkg, newver, version, elapsed)
    assert r.error is not None

    if isinstance(r.error, Exception):
      e = r.error
      build_logger.error(
        'failed', pkgbase = pkg,
        nv_version = newver, elapsed = elapsed, exc_info = e,
      )
      msg = repr(e)
      mod = repo.lilacinfos[pkg]

      l10n = intl.get_l10n('mail')
      if isinstance(e, MissingDependencies):
        faileddeps = e.deps.intersection(failed)
        # e.deps - faileddeps = failed previously
        failed[pkg] = tuple(e.deps)
        if e.deps == faileddeps:
          msg = l10n.format_value('dependency-issue-failed', {
            'pkg': pkg,
            'faileddeps': str(faileddeps),
            'count': len(faileddeps),
          })
        else:
          msg = l10n.format_value('dependency-issue-failed-this-batch', {
            'pkg': pkg,
            'deps': str(e.deps),
            'count_deps': len(e.deps),
            'faileddeps': str(faileddeps),
            'count_failed': len(faileddeps),
          })
        repo.send_error_report(
          mod,
          subject = l10n.format_value('dependency-issue-subject'),
          msg = msg,
        )
      else:
        repo.send_error_report(mod, exc=e, logfile=logfile)

    else:
      build_logger.error(
        'failed', pkgbase = pkg,
        nv_version = newver, pkg_version = version,
        elapsed = elapsed, error = r.error,
      )
      msg = r.error

  if db.USE:
    if r.rusage:
      cputime = r.rusage.cputime
      memory = r.rusage.memory
    else:
      cputime = memory = None
    reason_s = json.dumps([r.to_dict() for r in build_reasons[pkg]])
    with db.get_session() as s:
      maintainers = json.dumps(repo.lilacinfos[pkg].maintainers)
      s.execute(
        '''insert into pkglog
        (pkgbase, nv_version, pkg_version, elapsed, result, cputime, memory,
         msg, build_reasons, maintainers, builder) values
        (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)''',
        (pkg, newver, version, elapsed, r.__class__.__name__, cputime, memory,
         msg, reason_s, maintainers, wm.name))
      db.mark_pkg_as(s, pkg, 'done')
      db.build_updated(s)

  buildsorter.done(pkg)

  if r:
    built.add(pkg)
  elif pkg not in failed:
    failed[pkg] = ()

WORKER_NO = 0
WORKER_NO_LOCK = threading.Lock()

def setup_thread() -> None:
  global WORKER_NO
  with WORKER_NO_LOCK:
    TLS.worker_no = WORKER_NO
    WORKER_NO += 1

def build_nvchecker_reason(
  items: list[tuple[int, str]],
  results: NvResults,
) -> BuildReason.NvChecker:
  changes = []
  for i, _ in items:
    ver_change = results[i]
    changes.append((ver_change.oldver, ver_change.newver))
  return BuildReason.NvChecker(items, changes)

def get_workermans():
  from lilac2 import workerman

  ret = []

  remotes = [
    workerman.RemoteWorkerManager(remote)
    for remote in config.get('remoteworker', [])
    if remote.get('enabled', False)
  ]
  ret.extend(remotes)

  if not config['lilac'].get('disable_local_worker', False):
    max_concurrency = config['lilac'].get('max_concurrency', 1)
    local = workerman.LocalWorkerManager(max_concurrency)
    ret.append(local)

  workers_before = 0
  for wm in ret:
    wm.workers_before_me = workers_before
    workers_before += wm.max_concurrency

  return ret

def main_may_raise(
  D: dict[str, Any], pkgs_from_args: List[str], logdir: Path,
) -> None:
  global DEPMAP, BUILD_DEPMAP

  if get_git_branch() not in ['master', 'main']:
    raise Exception('repo not on master or main, aborting.')

  if os.path.islink('/etc/resolv.conf'):
    logger.warning('/etc/resolv.conf is a symlink; this might not work!')

  pacman_conf = config['misc'].get('pacman_conf')
  workermans = get_workermans()
  for wm in workermans:
    wm.prepare_batch(pacman_conf)

  if dburl := config['lilac'].get('dburl'):
    schema = config['lilac'].get('schema')
    db.setup(dburl, schema)

  if cmds := config.get('misc', {}).get('prerun'):
    for cmd in cmds:
      subprocess.check_call(cmd)

  git_reset_hard()
  git_pull_override()
  failed = REPO.load_managed_lilac_and_report()

  depman = DependencyManager(REPO.repodir)
  DEPMAP, BUILD_DEPMAP = get_dependency_map(depman, REPO.lilacinfos)

  failed_info = D.get('failed', {})

  U = set(REPO.lilacinfos)
  last_commit = D.get('last_commit', EMPTY_COMMIT)
  changed = get_changed_packages(last_commit, 'HEAD') & U

  failed_prev = set(failed_info.keys())
  # no update from upstream, but build instructions have changed; rebuild
  # failed ones
  need_rebuild_failed = failed_prev & changed
  # if pkgrel is updated, build a new release
  need_rebuild_pkgrel = {x for x in changed
                        if pkgrel_changed(last_commit, 'HEAD', x)}

  # packages we care about
  care_pkgs: set[str] = set()
  for p in pkgs_from_args:
    if ':' in p:
      pkg = p.split(':', 1)[0]
    else:
      pkg = p
    care_pkgs.update(dep.pkgname for dep in DEPMAP[pkg])
    care_pkgs.add(pkg)
    # make sure they have nvdata
    care_pkgs.update(need_rebuild_failed)
    care_pkgs.update(need_rebuild_pkgrel)

  proxy = config['nvchecker'].get('proxy')
  _nvdata, unknown, rebuild = packages_need_update(
    REPO, proxy, care_pkgs,
  )
  nvdata.update(_nvdata) # update to the global object

  need_rebuild_pkgrel -= unknown

  nv_changed = {}
  for p, vers in nvdata.items():
    diff_idxs = [i for i, v in enumerate(vers)
                 if v.oldver != v.newver]
    if diff_idxs:
      info = REPO.lilacinfos[p]
      confs = info.update_on
      sources = [(i, confs[i]['source']) for i in diff_idxs]
      nv_changed[p] = sources

  if db.USE:
    now = datetime.datetime.now().astimezone()
    pkgs_may_throttle = [p for p in nv_changed if REPO.lilacinfos[p].throttle_info]
    last_times = dict(db.get_pkgs_last_success_times(pkgs_may_throttle))
    for p, sources in nv_changed.items():
      ss = []
      for idx, source in sources:
        if interval := REPO.lilacinfos[p].throttle_info.get(idx):
          if last := last_times.get(p):
            if last + interval > now:
              continue
        ss.append((idx, source))
      if ss:
        nvc = build_nvchecker_reason(ss, nvdata[p])
        build_reasons[p].append(nvc)
  else:
    for p, sources in nv_changed.items():
      nvc = build_nvchecker_reason(sources, nvdata[p])
      build_reasons[p].append(nvc)

  if pkgs_from_args:
    for p in pkgs_from_args:
      if ':' in p:
        p, runner = p.split(':', 1)
      else:
        runner = None
      build_reasons[p].append(BuildReason.Cmdline(runner))

  for p in need_rebuild_pkgrel:
    build_reasons[p].append(BuildReason.UpdatedPkgrel())

  for p in need_rebuild_failed:
    build_reasons[p].append(BuildReason.UpdatedFailed())

  if not pkgs_from_args:
    for p, i in failed_info.items():
      # p might have been removed
      if p in REPO.lilacinfos and (deps := i['missing']):
        build_reasons[p].append(BuildReason.FailedByDeps(deps))

  if_this_then_those = defaultdict(set)
  for p, info in REPO.lilacinfos.items():
    for i in info.update_on_build:
      if_this_then_those[i.pkgbase].add(p)
  more_pkgs = set()
  for p in build_reasons:
    if pkgs := if_this_then_those.get(p):
      more_pkgs.update(pkgs)
  while True:
    add_to_more_pkgs = set()
    for p in more_pkgs:
      if pkgs := if_this_then_those.get(p):
        add_to_more_pkgs.update(pkgs)
    if add_to_more_pkgs.issubset(more_pkgs):
      break
    more_pkgs.update(add_to_more_pkgs)
  for p in more_pkgs:
    update_on_build = REPO.lilacinfos[p].update_on_build
    build_reasons[p].append(BuildReason.OnBuild(update_on_build))

  update_succeeded: set[str] = set()

  try:
    build_logger.info('build start')
    if db.USE:
      logdir_name = logdir.name
      with db.get_session() as s:
        s.execute('insert into batch (event, logdir) values (%s, %s)',
                  ('start', logdir_name))
        db.build_updated(s)
    start_build(REPO, logdir, failed, update_succeeded, workermans)
  finally:
    # fetch remote commits
    for wm in workermans:
      wm.finish_batch()

    D['last_commit'] = git_last_commit()
    # handle what has been processed even on exception
    for k, v in failed.items():
      if nv := nvdata.get(k):
        failed_info[k] = {
          'version': nv.newver, # not used
          'missing': v,
        }

    for x in update_succeeded:
      if x in failed_info:
        del failed_info[x]
    # cleanup removed package failed_info
    for x in tuple(failed_info.keys()):
      if x not in REPO.lilacinfos:
        del failed_info[x]
    D['failed'] = failed_info

    if config['lilac']['rebuild_failed_pkgs']:
      if update_succeeded:
        nvtake(update_succeeded, REPO.lilacinfos)
    else:
      updated_by_nv = {
        p for p, rs in build_reasons.items()
        if any(isinstance(r, BuildReason.NvChecker) for r in rs)
      }
      if updated_by_nv:
        # only nvtake packages we have tried to build (excluding unbuilt
        # packages due to internal errors)
        built = update_succeeded.union(failed)
        update_nv = built & updated_by_nv
        nvtake(update_nv, REPO.lilacinfos)

    build_logger.info('build end')
    if db.USE:
      with db.get_session() as s:
        s.execute('''insert into batch (event) values ('stop')''')
        db.build_updated(s)

    git_reset_hard()
    if config['lilac']['git_push']:
      git_push()

    if cmds := config.get('misc', {}).get('postrun'):
      for cmd in cmds:
        subprocess.check_call(cmd)

def main(logdir: Path, pkgs_from_args: List[str]) -> None:
  store = mydir / 'store'
  with PickledData(store, default={}) as D:
    try:
      main_may_raise(D, pkgs_from_args, logdir)
    except Exception:
      l10n = intl.get_l10n('main')
      tb = traceback.format_exc()
      logger.exception('unexpected error')
      subject = l10n.format_value('runtime-error')
      msg = l10n.format_value('runtime-error-traceback') + '\n\n' + tb
      REPO.report_error(subject, msg)

def setup() -> Path:
  prctl.set_child_subreaper(1)

  logdir = mydir / 'log' / time.strftime('%Y-%m-%dT%H:%M:%S')
  logdir.mkdir(parents=True, exist_ok=True)
  logfile = logdir / 'lilac-main.log'
  fd = os.open(logfile, os.O_WRONLY | os.O_CREAT, 0o644)
  os.dup2(fd, 1)
  os.dup2(fd, 2)
  os.close(fd)

  enable_pretty_logging('DEBUG')
  if 'MAKEFLAGS' not in os.environ:
    cores = os.process_cpu_count()
    if cores is not None:
      os.environ['MAKEFLAGS'] = '-j{0}'.format(cores)

  lock_file(mydir / '.lock')

  setup_build_logger()
  os.chdir(REPO.repodir)

  return logdir

if __name__ == '__main__':
  try:
    logdir = setup()
    main(logdir, sys.argv[1:])
  except Exception:
    logger.exception('unexpected error')
