setup/package.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 ##############################################################################
   4 #
   5 #    OpenERP, Open Source Management Solution
   6 #    Copyright (C) 2004-Today OpenERP SA (<http://www.openerp.com>).
   7 #
   8 #    This program is free software: you can redistribute it and/or modify
   9 #    it under the terms of the GNU Affero General Public License as
  10 #    published by the Free Software Foundation, either version 3 of the
  11 #    License, or (at your option) any later version.
  12 #
  13 #    This program is distributed in the hope that it will be useful,
  14 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #    GNU Affero General Public License for more details.
  17 #
  18 #    You should have received a copy of the GNU Affero General Public License
  19 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20 #
  21 ##############################################################################
  22
  23 import optparse
  24 import os
  25 import pexpect
  26 import shutil
  27 import signal
  28 import subprocess
  29 import tempfile
  30 import time
  31 import xmlrpclib
  32 from contextlib import contextmanager
  33 from glob import glob
  34 from os.path import abspath, dirname, join
  35 from tempfile import NamedTemporaryFile
  36
  37
  38 #----------------------------------------------------------
  39 # Utils
  40 #----------------------------------------------------------
  41 execfile(join(dirname(__file__), '..', 'openerp', 'release.py'))
  42 version = version.split('-')[0]
  43
  44 timestamp = time.strftime("%Y%m%d", time.gmtime())
  45 PUBLISH_DIRS = {
  46     'debian': 'deb',
  47     'redhat': 'rpm',
  48     'tarball': 'src',
  49     'windows': 'exe',
  50 }
  51 EXTENSIONS = [
  52     '.tar.gz',
  53     '.deb',
  54     '.dsc',
  55     '.changes',
  56     '.noarch.rpm',
  57     '.exe',
  58 ]
  59
  60 def mkdir(d):
  61     if not os.path.isdir(d):
  62         os.makedirs(d)
  63
  64 def system(l, chdir=None):
  65     print l
  66     if chdir:
  67         cwd = os.getcwd()
  68         os.chdir(chdir)
  69     if isinstance(l, list):
  70         rc = os.spawnvp(os.P_WAIT, l[0], l)
  71     elif isinstance(l, str):
  72         tmp = ['sh', '-c', l]
  73         rc = os.spawnvp(os.P_WAIT, tmp[0], tmp)
  74     if chdir:
  75         os.chdir(cwd)
  76     return rc
  77
  78 def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'):
  79     time.sleep(5)
  80     modules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  81         dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'installed')]
  82     )
  83     if modules and len(modules) > 1:
  84         time.sleep(1)
  85         toinstallmodules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  86             dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'to install')]
  87         )
  88         if toinstallmodules:
  89             print("Package test: FAILED. Not able to install dependencies of base.")
  90             raise Exception("Installation of package failed")
  91         else:
  92             print("Package test: successfuly installed %s modules" % len(modules))
  93     else:
  94         print("Package test: FAILED. Not able to install base.")
  95         raise Exception("Installation of package failed")
  96
  97 def publish(o, type, releases):
  98     def _publish(o, release):
  99         arch = ''
 100         filename = release.split(os.path.sep)[-1]
 101
 102         extension = None
 103         for EXTENSION in EXTENSIONS:
 104             if filename.endswith(EXTENSION):
 105                 extension = EXTENSION
 106                 filename = filename.replace(extension, '')
 107                 break
 108         if extension is None:
 109             raise Exception("Extension of %s is not handled" % filename)
 110
 111         # keep _all or _amd64
 112         if filename.count('_') > 1:
 113             arch = '_' + filename.split('_')[-1]
 114
 115         release_dir = PUBLISH_DIRS[type]
 116         release_filename = 'odoo_%s.%s%s%s' % (version, timestamp, arch, extension)
 117         release_path = join(o.pub, release_dir, release_filename)
 118
 119         system('mkdir -p %s' % join(o.pub, release_dir))
 120         shutil.move(join(o.build_dir, release), release_path)
 121
 122         # Latest/symlink handler
 123         release_abspath = abspath(release_path)
 124         latest_abspath = release_abspath.replace(timestamp, 'latest')
 125
 126         if os.path.islink(latest_abspath):
 127             os.unlink(latest_abspath)
 128
 129         os.symlink(release_abspath, latest_abspath)
 130
 131         return release_path
 132
 133     published = []
 134     if isinstance(releases, basestring):
 135         published.append(_publish(o, releases))
 136     elif isinstance(releases, list):
 137         for release in releases:
 138             published.append(_publish(o, release))
 139     return published
 140
 141 class OdooDocker(object):
 142     def __init__(self):
 143         self.log_file = NamedTemporaryFile(mode='w+b', prefix="bash", suffix=".txt", delete=False)
 144         self.port = 8069  # TODO sle: reliable way to get a free port?
 145         self.prompt_re = '(\r\nroot@|bash-).*# '
 146         self.timeout = 600
 147
 148     def system(self, command):
 149         self.docker.sendline(command)
 150         self.docker.expect(self.prompt_re)
 151
 152     def start(self, docker_image, build_dir, pub_dir):
 153         self.build_dir = build_dir
 154         self.pub_dir = pub_dir
 155
 156         self.docker = pexpect.spawn(
 157             'docker run -v %s:/opt/release -p 127.0.0.1:%s:8069'
 158             ' -t -i %s /bin/bash --noediting' % (self.build_dir, self.port, docker_image),
 159             timeout=self.timeout
 160         )
 161         time.sleep(2)  # let the bash start
 162         self.docker.logfile_read = self.log_file
 163         self.id = subprocess.check_output('docker ps -l -q', shell=True)
 164
 165     def end(self):
 166         try:
 167             _rpc_count_modules(port=str(self.port))
 168         except Exception, e:
 169             print('Exception during docker execution: %s:' % str(e))
 170             print('Error during docker execution: printing the bash output:')
 171             with open(self.log_file.name) as f:
 172                 print '\n'.join(f.readlines())
 173             raise
 174         finally:
 175             self.docker.close()
 176             system('docker rm -f %s' % self.id)
 177             self.log_file.close()
 178             os.remove(self.log_file.name)
 179
 180 @contextmanager
 181 def docker(docker_image, build_dir, pub_dir):
 182     _docker = OdooDocker()
 183     try:
 184         _docker.start(docker_image, build_dir, pub_dir)
 185         try:
 186             yield _docker
 187         except Exception, e:
 188             raise
 189     finally:
 190         _docker.end()
 191
 192 class KVM(object):
 193     def __init__(self, o, image, ssh_key='', login='openerp'):
 194         self.o = o
 195         self.image = image
 196         self.ssh_key = ssh_key
 197         self.login = login
 198
 199     def timeout(self,signum,frame):
 200         print "vm timeout kill",self.pid
 201         os.kill(self.pid,15)
 202
 203     def start(self):
 204         l="kvm -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -drive".split(" ")
 205         #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image)
 206         l.append('file=%s,snapshot=on'%self.image)
 207         #l.extend(['-vnc','127.0.0.1:1'])
 208         l.append('-nographic')
 209         print " ".join(l)
 210         self.pid=os.spawnvp(os.P_NOWAIT, l[0], l)
 211         time.sleep(10)
 212         signal.alarm(2400)
 213         signal.signal(signal.SIGALRM, self.timeout)
 214         try:
 215             self.run()
 216         finally:
 217             signal.signal(signal.SIGALRM, signal.SIG_DFL)
 218             os.kill(self.pid,15)
 219             time.sleep(10)
 220
 221     def ssh(self,cmd):
 222         l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd]
 223         system(l)
 224
 225     def rsync(self,args,options='--delete --exclude .bzrignore'):
 226         cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args)
 227         system(cmd)
 228
 229     def run(self):
 230         pass
 231
 232 class KVMWinBuildExe(KVM):
 233     def run(self):
 234         with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
 235             f.write("VERSION=%s\n" % self.o.version_full)
 236         with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
 237             f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', ''))
 238
 239         self.ssh("mkdir -p build")
 240         self.rsync('%s/ %s@127.0.0.1:build/server/' % (self.o.build_dir, self.login))
 241         self.ssh("cd build/server/setup/win32;time make allinone;")
 242         self.rsync('%s@127.0.0.1:build/server/setup/win32/release/ %s/' % (self.login, self.o.build_dir), '')
 243         print "KVMWinBuildExe.run(): done"
 244
 245 class KVMWinTestExe(KVM):
 246     def run(self):
 247         # Cannot use o.version_full when the version is not correctly parsed
 248         # (for instance, containing *rc* or *dev*)
 249         setuppath = glob("%s/openerp-server-setup-*.exe" % self.o.build_dir)[0]
 250         setupfile = setuppath.split('/')[-1]
 251         setupversion = setupfile.split('openerp-server-setup-')[1].split('.exe')[0]
 252
 253         self.rsync('"%s" %s@127.0.0.1:' % (setuppath, self.login))
 254         self.ssh("TEMP=/tmp ./%s /S" % setupfile)
 255         self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"Odoo %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion)
 256         self.ssh('/cygdrive/c/"Program Files"/"Odoo %s"/server/openerp-server.exe -d mycompany -i base --stop-after-init' % setupversion)
 257         self.ssh('net start odoo-server-8.0')
 258         _rpc_count_modules(port=18069)
 259
 260 #----------------------------------------------------------
 261 # Stage: building
 262 #----------------------------------------------------------
 263 def _prepare_build_dir(o):
 264     cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
 265     system(cmd + ['%s/' % o.odoo_dir, o.build_dir])
 266     for i in glob(join(o.build_dir, 'addons/*')):
 267         shutil.move(i, join(o.build_dir, 'openerp/addons'))
 268
 269 def build_tgz(o):
 270     system(['python2', 'setup.py', '--quiet', 'sdist'], o.build_dir)
 271     system(['cp', glob('%s/dist/odoo-*.tar.gz' % o.build_dir)[0], '%s/odoo.tar.gz' % o.build_dir])
 272
 273 def build_deb(o):
 274     system(['dpkg-buildpackage', '-rfakeroot'], o.build_dir)
 275     system(['mv', glob('%s/../odoo_*.deb' % o.build_dir)[0], '%s' % o.build_dir])
 276     system(['mv', glob('%s/../odoo_*.dsc' % o.build_dir)[0], '%s' % o.build_dir])
 277     system(['mv', glob('%s/../odoo_*_amd64.changes' % o.build_dir)[0], '%s' % o.build_dir])
 278     system(['mv', glob('%s/../odoo_*.tar.gz' % o.build_dir)[0], '%s' % o.build_dir])
 279
 280 def build_rpm(o):
 281     system(['python2', 'setup.py', '--quiet', 'bdist_rpm'], o.build_dir)
 282     system(['cp', glob('%s/dist/odoo-*.noarch.rpm' % o.build_dir)[0], '%s/odoo.noarch.rpm' % o.build_dir])
 283
 284 def build_exe(o):
 285     KVMWinBuildExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 286     system(['cp', glob('%s/openerp*.exe' % o.build_dir)[0], '%s/odoo.exe' % o.build_dir])
 287
 288 #----------------------------------------------------------
 289 # Stage: testing
 290 #----------------------------------------------------------
 291 def test_tgz(o):
 292     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 293         wheezy.release = 'odoo.tar.gz'
 294         wheezy.system('apt-get update -qq && apt-get upgrade -qq -y')
 295         wheezy.system("apt-get install postgresql python-dev postgresql-server-dev-all python-pip build-essential libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libssl-dev libjpeg-dev -y")
 296         wheezy.system("service postgresql start")
 297         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 298         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 299         wheezy.system('pip install -r /opt/release/requirements.txt')
 300         wheezy.system('/usr/local/bin/pip install /opt/release/%s' % wheezy.release)
 301         wheezy.system("useradd --system --no-create-home odoo")
 302         wheezy.system('su postgres -s /bin/bash -c "createuser -s odoo"')
 303         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 304         wheezy.system('mkdir /var/lib/odoo')
 305         wheezy.system('chown odoo:odoo /var/lib/odoo')
 306         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany -i base --stop-after-init"')
 307         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany &"')
 308
 309 def test_deb(o):
 310     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 311         wheezy.release = '*.deb'
 312         wheezy.system('/usr/bin/apt-get update -qq && /usr/bin/apt-get upgrade -qq -y')
 313         wheezy.system("apt-get install postgresql -y")
 314         wheezy.system("service postgresql start")
 315         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 316         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 317         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 318         wheezy.system('/usr/bin/dpkg -i /opt/release/%s' % wheezy.release)
 319         wheezy.system('/usr/bin/apt-get install -f -y')
 320         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 321         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany &"')
 322
 323 def test_rpm(o):
 324     with docker('centos:centos7', o.build_dir, o.pub) as centos7:
 325         centos7.release = 'odoo.noarch.rpm'
 326         # Dependencies
 327         centos7.system('yum install -d 0 -e 0 epel-release -y')
 328         centos7.system('yum update -d 0 -e 0 -y')
 329         # Manual install/start of postgres
 330         centos7.system('yum install -d 0 -e 0 postgresql postgresql-server postgresql-libs postgresql-contrib postgresql-devel -y')
 331         centos7.system('mkdir -p /var/lib/postgres/data')
 332         centos7.system('chown -R postgres:postgres /var/lib/postgres/data')
 333         centos7.system('chmod 0700 /var/lib/postgres/data')
 334         centos7.system('su postgres -c "initdb -D /var/lib/postgres/data -E UTF-8"')
 335         centos7.system('cp /usr/share/pgsql/postgresql.conf.sample /var/lib/postgres/data/postgresql.conf')
 336         centos7.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
 337         centos7.system('sleep 5')
 338         centos7.system('su postgres -c "createdb mycompany"')
 339         # Odoo install
 340         centos7.system('yum install -d 0 -e 0 /opt/release/%s -y' % centos7.release)
 341         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 342         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany &"')
 343
 344 def test_exe(o):
 345     KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 346
 347 #---------------------------------------------------------
 348 # Generates Packages, Sources and Release files of debian package
 349 #---------------------------------------------------------
 350 def gen_deb_package(o, published_files):
 351     # Executes command to produce file_name in path, and moves it to o.pub/deb
 352     def _gen_file(o, (command, file_name), path):
 353         cur_tmp_file_path = os.path.join(path, file_name)
 354         with open(cur_tmp_file_path, 'w') as out:
 355             subprocess.call(command, stdout=out, cwd=path)
 356         system(['cp', cur_tmp_file_path, os.path.join(o.pub, 'deb', file_name)])
 357
 358     # Copy files to a temp directory (required because the working directory must contain only the
 359     # files of the last release)
 360     temp_path = tempfile.mkdtemp(suffix='debPackages')
 361     for pub_file_path in published_files:
 362         system(['cp', pub_file_path, temp_path])
 363
 364     commands = [
 365         (['dpkg-scanpackages', '.'], "Packages"),  # Generate Packages file
 366         (['dpkg-scansources', '.'], "Sources"),  # Generate Sources file
 367         (['apt-ftparchive', 'release', '.'], "Release")  # Generate Release file
 368     ]
 369     # Generate files
 370     for command in commands:
 371         _gen_file(o, command, temp_path)
 372     # Remove temp directory
 373     shutil.rmtree(temp_path)
 374
 375     # Generate Release.gpg (= signed Release)
 376     # Options -abs: -a (Create ASCII armored output), -b (Make a detach signature), -s (Make a signature)
 377     subprocess.call(['gpg', '--yes', '-abs', '-o', 'Release.gpg', 'Release'], cwd=os.path.join(o.pub, 'deb'))
 378
 379 #---------------------------------------------------------
 380 # Generates an RPM repo
 381 #---------------------------------------------------------
 382 def gen_rpm_repo(o, file_name):
 383     # Sign the RPM
 384     subprocess.call(['rpm', '--resign', file_name], cwd=os.path.join(o.pub, 'rpm'))
 385
 386     # Removes the old repodata
 387     subprocess.call(['rm', '-rf', os.path.join(o.pub, 'rpm', 'repodata')])
 388
 389     # Copy files to a temp directory (required because the working directory must contain only the
 390     # files of the last release)
 391     temp_path = tempfile.mkdtemp(suffix='rpmPackages')
 392     subprocess.call(['cp', file_name, temp_path])
 393
 394     subprocess.call(['createrepo', temp_path])  # creates a repodata folder in temp_path
 395     subprocess.call(['cp', '-r', os.path.join(temp_path, "repodata"), os.path.join(o.pub, 'rpm')])
 396
 397     # Remove temp directory
 398     shutil.rmtree(temp_path)
 399
 400 #----------------------------------------------------------
 401 # Options and Main
 402 #----------------------------------------------------------
 403 def options():
 404     op = optparse.OptionParser()
 405     root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 406     build_dir = "%s-%s" % (root, timestamp)
 407
 408     op.add_option("-b", "--build-dir", default=build_dir, help="build directory (%default)", metavar="DIR")
 409     op.add_option("-p", "--pub", default=None, help="pub directory (%default)", metavar="DIR")
 410     op.add_option("", "--no-testing", action="store_true", help="don't test the builded packages")
 411     op.add_option("-v", "--version", default='8.0', help="version (%default)")
 412
 413     op.add_option("", "--no-debian", action="store_true", help="don't build the debian package")
 414     op.add_option("", "--no-rpm", action="store_true", help="don't build the rpm package")
 415     op.add_option("", "--no-tarball", action="store_true", help="don't build the tarball")
 416     op.add_option("", "--no-windows", action="store_true", help="don't build the windows package")
 417
 418     # Windows VM
 419     op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/winxp27/winxp27.vdi', help="%default")
 420     op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/winxp27/id_rsa', help="%default")
 421     op.add_option("", "--vm-winxp-login", default='Naresh', help="Windows login (%default)")
 422     op.add_option("", "--vm-winxp-python-version", default='2.7', help="Windows Python version installed in the VM (default: %default)")
 423
 424     (o, args) = op.parse_args()
 425     # derive other options
 426     o.odoo_dir = root
 427     o.pkg = join(o.build_dir, 'pkg')
 428     o.version_full = '%s-%s' % (o.version, timestamp)
 429     o.work = join(o.build_dir, 'openerp-%s' % o.version_full)
 430     o.work_addons = join(o.work, 'openerp', 'addons')
 431
 432     return o
 433
 434 def main():
 435     o = options()
 436     _prepare_build_dir(o)
 437     try:
 438         if not o.no_tarball:
 439             build_tgz(o)
 440             try:
 441                 if not o.no_testing:
 442                     test_tgz(o)
 443                 published_files = publish(o, 'tarball', ['odoo.tar.gz'])
 444             except Exception, e:
 445                 print("Won't publish the tgz release.\n Exception: %s" % str(e))
 446         if not o.no_debian:
 447             build_deb(o)
 448             try:
 449                 if not o.no_testing:
 450                     test_deb(o)
 451
 452                 to_publish = []
 453                 to_publish.append(glob("%s/odoo_*.deb" % o.build_dir)[0])
 454                 to_publish.append(glob("%s/odoo_*.dsc" % o.build_dir)[0])
 455                 to_publish.append(glob("%s/odoo_*.changes" % o.build_dir)[0])
 456                 to_publish.append(glob("%s/odoo_*.tar.gz" % o.build_dir)[0])
 457                 published_files = publish(o, 'debian', to_publish)
 458                 gen_deb_package(o, published_files)
 459             except Exception, e:
 460                 print("Won't publish the deb release.\n Exception: %s" % str(e))
 461         if not o.no_rpm:
 462             build_rpm(o)
 463             try:
 464                 if not o.no_testing:
 465                     test_rpm(o)
 466                 published_files = publish(o, 'redhat', ['odoo.noarch.rpm'])
 467                 gen_rpm_repo(o, published_files[0])
 468             except Exception, e:
 469                 print("Won't publish the rpm release.\n Exception: %s" % str(e))
 470         if not o.no_windows:
 471             build_exe(o)
 472             try:
 473                 if not o.no_testing:
 474                     test_exe(o)
 475                 published_files = publish(o, 'windows', ['odoo.exe'])
 476             except Exception, e:
 477                 print("Won't publish the exe release.\n Exception: %s" % str(e))
 478     except:
 479         pass
 480     finally:
 481         shutil.rmtree(o.build_dir)
 482         print('Build dir %s removed' % o.build_dir)
 483
 484         if not o.no_testing:
 485             system("docker rm -f `docker ps -a | awk '{print $1 }'` 2>>/dev/null")
 486             print('Remaining dockers removed')
 487
 488
 489 if __name__ == '__main__':
 490     main()