setup/package.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 ##############################################################################
   4 #
   5 #    OpenERP, Open Source Management Solution
   6 #    Copyright (C) 2004-Today OpenERP SA (<http://www.openerp.com>).
   7 #
   8 #    This program is free software: you can redistribute it and/or modify
   9 #    it under the terms of the GNU Affero General Public License as
  10 #    published by the Free Software Foundation, either version 3 of the
  11 #    License, or (at your option) any later version.
  12 #
  13 #    This program is distributed in the hope that it will be useful,
  14 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #    GNU Affero General Public License for more details.
  17 #
  18 #    You should have received a copy of the GNU Affero General Public License
  19 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20 #
  21 ##############################################################################
  22
  23 import optparse
  24 import os
  25 import pexpect
  26 import shutil
  27 import signal
  28 import time
  29 import xmlrpclib
  30 from contextlib import contextmanager
  31 from glob import glob
  32 from os.path import abspath, dirname, join
  33 from subprocess import check_output
  34 from tempfile import NamedTemporaryFile
  35
  36
  37 #----------------------------------------------------------
  38 # Utils
  39 #----------------------------------------------------------
  40 execfile(join(dirname(__file__), '..', 'openerp', 'release.py'))
  41 timestamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
  42 PUBLISH_DIRS = {
  43     'tar.gz': 'src',
  44     'exe': 'exe',
  45     'deb': 'deb',
  46     'dsc': 'deb',
  47     'changes': 'deb',
  48     'deb.tar.gz': ['deb', 'tar.gz'],
  49     'noarch.rpm': 'rpm',
  50     'src.rpm': 'rpm',
  51 }
  52
  53 def mkdir(d):
  54     if not os.path.isdir(d):
  55         os.makedirs(d)
  56
  57 def system(l, chdir=None):
  58     print l
  59     if chdir:
  60         cwd = os.getcwd()
  61         os.chdir(chdir)
  62     if isinstance(l, list):
  63         rc = os.spawnvp(os.P_WAIT, l[0], l)
  64     elif isinstance(l, str):
  65         tmp = ['sh', '-c', l]
  66         rc = os.spawnvp(os.P_WAIT, tmp[0], tmp)
  67     if chdir:
  68         os.chdir(cwd)
  69     return rc
  70
  71 def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'):
  72     time.sleep(5)
  73     modules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  74         dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'installed')]
  75     )
  76     if modules:
  77         print("Package test: successfuly installed %s modules" % len(modules))
  78     else:
  79         raise Exception("Installation of package failed")
  80
  81 def publish(o, releases):
  82     def _publish(o, release):
  83         extension = ''.join(release.split('.', 1)[1])
  84         release_extension = PUBLISH_DIRS[extension][1] if isinstance(PUBLISH_DIRS[extension], list) else extension
  85         release_dir = PUBLISH_DIRS[extension][0] if isinstance(PUBLISH_DIRS[extension], list) else PUBLISH_DIRS[extension]
  86
  87         release_filename = 'odoo_%s-%s.%s' % (version, timestamp, release_extension)
  88         release_path = join(o.pub, release_dir, release_filename)
  89
  90         system('mkdir -p %s' % join(o.pub, release_dir))
  91         shutil.move(join(o.build_dir, release), release_path)
  92
  93         # Latest/symlink handler
  94         release_abspath = abspath(release_path)
  95         latest_abspath = release_abspath.replace(timestamp, 'latest')
  96
  97         if os.path.islink(latest_abspath):
  98             os.unlink(latest_abspath)
  99
 100         os.symlink(release_abspath, latest_abspath)
 101
 102     if isinstance(releases, basestring):
 103         _publish(o, releases)
 104     elif isinstance(releases, list):
 105         for release in releases:
 106             _publish(o, release)
 107
 108 class OdooDocker(object):
 109     def __init__(self):
 110         self.log_file = NamedTemporaryFile(mode='w+b', prefix="bash", suffix=".txt", delete=False)
 111         self.port = 8069  # TODO sle: reliable way to get a free port?
 112         self.prompt_re = '(\r\nroot@|bash-).*# '
 113         self.timeout = 600
 114
 115     def system(self, command):
 116         self.docker.sendline(command)
 117         self.docker.expect(self.prompt_re)
 118
 119     def start(self, docker_image, build_dir, pub_dir):
 120         self.build_dir = build_dir
 121         self.pub_dir = pub_dir
 122
 123         self.docker = pexpect.spawn(
 124             'docker run -v %s:/opt/release -p 127.0.0.1:%s:8069'
 125             ' -t -i %s /bin/bash --noediting' % (self.build_dir, self.port, docker_image),
 126             timeout=self.timeout
 127         )
 128         time.sleep(2)  # let the bash start
 129         self.docker.logfile_read = self.log_file
 130         self.id = check_output('docker ps -l -q', shell=True)
 131
 132     def end(self):
 133         try:
 134             _rpc_count_modules(port=str(self.port))
 135         except Exception, e:
 136             print('Exception during docker execution: %s:' % str(e))
 137             print('Error during docker execution: printing the bash output:')
 138             with open(self.log_file.name) as f:
 139                 print '\n'.join(f.readlines())
 140             raise
 141         finally:
 142             self.docker.close()
 143             system('docker rm -f %s' % self.id)
 144             self.log_file.close()
 145             os.remove(self.log_file.name)
 146
 147 @contextmanager
 148 def docker(docker_image, build_dir, pub_dir):
 149     _docker = OdooDocker()
 150     try:
 151         _docker.start(docker_image, build_dir, pub_dir)
 152         try:
 153             yield _docker
 154         except Exception, e:
 155             raise
 156     finally:
 157         _docker.end()
 158
 159 class KVM(object):
 160     def __init__(self, o, image, ssh_key='', login='openerp'):
 161         self.o = o
 162         self.image = image
 163         self.ssh_key = ssh_key
 164         self.login = login
 165
 166     def timeout(self,signum,frame):
 167         print "vm timeout kill",self.pid
 168         os.kill(self.pid,15)
 169
 170     def start(self):
 171         l="kvm -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -drive".split(" ")
 172         #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image)
 173         l.append('file=%s,snapshot=on'%self.image)
 174         #l.extend(['-vnc','127.0.0.1:1'])
 175         l.append('-nographic')
 176         print " ".join(l)
 177         self.pid=os.spawnvp(os.P_NOWAIT, l[0], l)
 178         time.sleep(10)
 179         signal.alarm(2400)
 180         signal.signal(signal.SIGALRM, self.timeout)
 181         try:
 182             self.run()
 183         finally:
 184             signal.signal(signal.SIGALRM, signal.SIG_DFL)
 185             os.kill(self.pid,15)
 186             time.sleep(10)
 187
 188     def ssh(self,cmd):
 189         l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd]
 190         system(l)
 191
 192     def rsync(self,args,options='--delete --exclude .bzrignore'):
 193         cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args)
 194         system(cmd)
 195
 196     def run(self):
 197         pass
 198
 199 class KVMWinBuildExe(KVM):
 200     def run(self):
 201         with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
 202             f.write("VERSION=%s\n" % self.o.version_full)
 203         with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
 204             f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', ''))
 205
 206         self.ssh("mkdir -p build")
 207         self.rsync('%s/ %s@127.0.0.1:build/server/' % (self.o.build_dir, self.login))
 208         self.ssh("cd build/server/setup/win32;time make allinone;")
 209         self.rsync('%s@127.0.0.1:build/server/setup/win32/release/ %s/' % (self.login, self.o.build_dir), '')
 210         print "KVMWinBuildExe.run(): done"
 211
 212 class KVMWinTestExe(KVM):
 213     def run(self):
 214         # Cannot use o.version_full when the version is not correctly parsed
 215         # (for instance, containing *rc* or *dev*)
 216         setuppath = glob("%s/openerp-server-setup-*.exe" % self.o.build_dir)[0]
 217         setupfile = setuppath.split('/')[-1]
 218         setupversion = setupfile.split('openerp-server-setup-')[1].split('.exe')[0]
 219
 220         self.rsync('"%s" %s@127.0.0.1:' % (setuppath, self.login))
 221         self.ssh("TEMP=/tmp ./%s /S" % setupfile)
 222         self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"OpenERP %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion)
 223         self.ssh('/cygdrive/c/"Program Files"/"OpenERP %s"/server/openerp-server.exe -d mycompany -i base --stop-after-init' % setupversion)
 224         self.ssh(['/cygdrive/c/"Program Files"/"OpenERP %s"/server/openerp-server.exe -d mycompany &' % setupversion, '&'])
 225         _rpc_count_modules(port=18069)
 226
 227 #----------------------------------------------------------
 228 # Stage: building
 229 #----------------------------------------------------------
 230 def _prepare_build_dir(o):
 231     cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
 232     system(cmd + ['%s/' % o.odoo_dir, o.build_dir])
 233     for i in glob(join(o.build_dir, 'addons/*')):
 234         shutil.move(i, join(o.build_dir, 'openerp/addons'))
 235
 236 def build_tgz(o):
 237     system(['python2', 'setup.py', '--quiet', 'sdist'], o.build_dir)
 238     system(['cp', glob('%s/dist/openerp-*.tar.gz' % o.build_dir)[0], '%s/odoo.tar.gz' % o.build_dir])
 239
 240 def build_deb(o):
 241     system(['dpkg-buildpackage', '-rfakeroot', '-uc', '-us'], o.build_dir)
 242     system(['cp', glob('%s/../openerp_*.deb' % o.build_dir)[0], '%s/odoo.deb' % o.build_dir])
 243     system(['cp', glob('%s/../openerp_*.dsc' % o.build_dir)[0], '%s/odoo.dsc' % o.build_dir])
 244     system(['cp', glob('%s/../openerp_*_amd64.changes' % o.build_dir)[0], '%s/odoo_amd64.changes' % o.build_dir])
 245     system(['cp', glob('%s/../openerp_*.tar.gz' % o.build_dir)[0], '%s/odoo.deb.tar.gz' % o.build_dir])
 246
 247 def build_rpm(o):
 248     system(['python2', 'setup.py', '--quiet', 'bdist_rpm'], o.build_dir)
 249     system(['cp', glob('%s/dist/openerp-*.noarch.rpm' % o.build_dir)[0], '%s/odoo.noarch.rpm' % o.build_dir])
 250     system(['cp', glob('%s/dist/openerp-*.src.rpm' % o.build_dir)[0], '%s/odoo.src.rpm' % o.build_dir])
 251
 252 def build_exe(o):
 253     KVMWinBuildExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 254     system(['cp', glob('%s/openerp*.exe' % o.build_dir)[0], '%s/odoo.exe' % o.build_dir])
 255
 256 #----------------------------------------------------------
 257 # Stage: testing
 258 #----------------------------------------------------------
 259 def test_tgz(o):
 260     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 261         wheezy.release = 'odoo.tar.gz'
 262         wheezy.system('apt-get update -qq && apt-get upgrade -qq -y')
 263         wheezy.system("apt-get install postgresql python-dev postgresql-server-dev-all python-pip build-essential libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libssl-dev libjpeg-dev -y")
 264         wheezy.system("service postgresql start")
 265         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 266         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 267         wheezy.system('pip install -r /opt/release/requirements.txt')
 268         wheezy.system('/usr/local/bin/pip install /opt/release/%s' % wheezy.release)
 269         wheezy.system("useradd --system --no-create-home odoo")
 270         wheezy.system('su postgres -s /bin/bash -c "createuser -s odoo"')
 271         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 272         wheezy.system('mkdir /var/lib/odoo')
 273         wheezy.system('chown odoo:odoo /var/lib/odoo')
 274         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany -i base --stop-after-init"')
 275         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany &"')
 276
 277 def test_deb(o):
 278     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 279         wheezy.release = 'odoo.deb'
 280         wheezy.system('/usr/bin/apt-get update -qq && /usr/bin/apt-get upgrade -qq -y')
 281         wheezy.system("apt-get install postgresql -y")
 282         wheezy.system("service postgresql start")
 283         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 284         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 285         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 286         wheezy.system('/usr/bin/dpkg -i /opt/release/%s' % wheezy.release)
 287         wheezy.system('/usr/bin/apt-get install -f -y')
 288         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 289         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany &"')
 290
 291 def test_rpm(o):
 292     with docker('centos:centos7', o.build_dir, o.pub) as centos7:
 293         centos7.release = 'odoo.noarch.rpm'
 294         centos7.system('rpm -Uvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-1.noarch.rpm')
 295         centos7.system('yum update -y && yum upgrade -y')
 296         centos7.system('yum install python-pip gcc python-devel -y')
 297         centos7.system('pip install pydot pyPdf vatnumber xlwt http://download.gna.org/pychart/PyChart-1.39.tar.gz')
 298         centos7.system('yum install postgresql postgresql-server postgresql-libs postgresql-contrib postgresql-devel -y')
 299         centos7.system('mkdir -p /var/lib/postgres/data')
 300         centos7.system('chown -R postgres:postgres /var/lib/postgres/data')
 301         centos7.system('chmod 0700 /var/lib/postgres/data')
 302         centos7.system('su postgres -c "initdb -D /var/lib/postgres/data -E UTF-8"')
 303         centos7.system('cp /usr/share/pgsql/postgresql.conf.sample /var/lib/postgres/data/postgresql.conf')
 304         centos7.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
 305         centos7.system('su postgres -c "createdb mycompany"')
 306         centos7.system('export PYTHONPATH=${PYTHONPATH}:/usr/local/lib/python2.7/dist-packages')
 307         centos7.system('su postgres -c "createdb mycompany"')
 308         centos7.system('yum install /opt/release/%s -y' % centos7.release)
 309         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 310         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany &"')
 311
 312 def test_exe(o):
 313     KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 314
 315 #----------------------------------------------------------
 316 # Options and Main
 317 #----------------------------------------------------------
 318 def options():
 319     op = optparse.OptionParser()
 320     root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 321     build_dir = "%s-%s" % (root, timestamp)
 322
 323     op.add_option("-b", "--build-dir", default=build_dir, help="build directory (%default)", metavar="DIR")
 324     op.add_option("-p", "--pub", default=None, help="pub directory (%default)", metavar="DIR")
 325     op.add_option("", "--no-testing", action="store_true", help="don't test the builded packages")
 326     op.add_option("-v", "--version", default='8.0', help="version (%default)")
 327
 328     op.add_option("", "--no-debian", action="store_true", help="don't build the debian package")
 329     op.add_option("", "--no-rpm", action="store_true", help="don't build the rpm package")
 330     op.add_option("", "--no-tarball", action="store_true", help="don't build the tarball")
 331     op.add_option("", "--no-windows", action="store_true", help="don't build the windows package")
 332
 333     # Windows VM
 334     op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/winxp27/winxp27.vdi', help="%default")
 335     op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/winxp27/id_rsa', help="%default")
 336     op.add_option("", "--vm-winxp-login", default='Naresh', help="Windows login (%default)")
 337     op.add_option("", "--vm-winxp-python-version", default='2.7', help="Windows Python version installed in the VM (default: %default)")
 338
 339     (o, args) = op.parse_args()
 340     # derive other options
 341     o.odoo_dir = root
 342     o.pkg = join(o.build_dir, 'pkg')
 343     o.version_full = '%s-%s' % (o.version, timestamp)
 344     o.work = join(o.build_dir, 'openerp-%s' % o.version_full)
 345     o.work_addons = join(o.work, 'openerp', 'addons')
 346     return o
 347
 348 def main():
 349     o = options()
 350     _prepare_build_dir(o)
 351
 352     try:
 353         if not o.no_tarball:
 354             build_tgz(o)
 355             if not o.no_testing:
 356                 try:
 357                     test_tgz(o)
 358                     publish(o, 'odoo.tar.gz')
 359                 except Exception, e:
 360                     print("Won't publish the tgz release.\n Exception: %s" % str(e))
 361         if not o.no_debian:
 362             build_deb(o)
 363             if not o.no_testing:
 364                 try:
 365                     test_deb(o)
 366                     publish(o, ['odoo.deb', 'odoo.dsc', 'odoo_amd64.changes', 'odoo.deb.tar.gz'])
 367                     system('dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz', join(o.pub, 'deb'))
 368                 except Exception, e:
 369                     print("Won't publish the deb release.\n Exception: %s" % str(e))
 370         if not o.no_rpm:
 371             build_rpm(o)
 372             if not o.no_testing:
 373                 try:
 374                     test_rpm(o)
 375                     publish(o, ['odoo.noarch.rpm', 'odoo.src.rpm'])
 376                 except Exception, e:
 377                     print("Won't publish the rpm release.\n Exception: %s" % str(e))
 378         if not o.no_windows:
 379             build_exe(o)
 380             if not o.no_testing:
 381                 try:
 382                     test_exe(o)
 383                     publish(o, 'odoo.exe')
 384                 except Exception, e:
 385                     print("Won't publish the exe release.\n Exception: %s" % str(e))
 386     except:
 387         pass
 388     finally:
 389         for leftover in glob('%s/../openerp_*' % o.build_dir):
 390             os.remove(leftover)
 391
 392         shutil.rmtree(o.build_dir)
 393         print('Build dir %s removed' % o.build_dir)
 394
 395         if not o.no_testing:
 396             system("docker rm -f `docker ps -a | awk '{print $1 }'` 2>>/dev/null")
 397             print('Remaining dockers removed')
 398
 399
 400 if __name__ == '__main__':
 401     main()