setup/package.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 ##############################################################################
   4 #
   5 #    OpenERP, Open Source Management Solution
   6 #    Copyright (C) 2004-Today OpenERP SA (<http://www.openerp.com>).
   7 #
   8 #    This program is free software: you can redistribute it and/or modify
   9 #    it under the terms of the GNU Affero General Public License as
  10 #    published by the Free Software Foundation, either version 3 of the
  11 #    License, or (at your option) any later version.
  12 #
  13 #    This program is distributed in the hope that it will be useful,
  14 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #    GNU Affero General Public License for more details.
  17 #
  18 #    You should have received a copy of the GNU Affero General Public License
  19 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20 #
  21 ##############################################################################
  22
  23 import optparse
  24 import os
  25 import pexpect
  26 import shutil
  27 import signal
  28 import subprocess
  29 import tempfile
  30 import time
  31 import xmlrpclib
  32 from contextlib import contextmanager
  33 from glob import glob
  34 from os.path import abspath, dirname, join
  35 from tempfile import NamedTemporaryFile
  36
  37
  38 #----------------------------------------------------------
  39 # Utils
  40 #----------------------------------------------------------
  41 execfile(join(dirname(__file__), '..', 'openerp', 'release.py'))
  42 version = version.split('-')[0]
  43
  44 timestamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
  45 PUBLISH_DIRS = {
  46     'tar.gz': 'src',
  47     'exe': 'exe',
  48     'deb': 'deb',
  49     'dsc': 'deb',
  50     'changes': 'deb',
  51     'deb.tar.gz': ['deb', 'tar.gz'],
  52     'noarch.rpm': 'rpm',
  53     'src.rpm': 'rpm',
  54 }
  55
  56 def mkdir(d):
  57     if not os.path.isdir(d):
  58         os.makedirs(d)
  59
  60 def system(l, chdir=None):
  61     print l
  62     if chdir:
  63         cwd = os.getcwd()
  64         os.chdir(chdir)
  65     if isinstance(l, list):
  66         rc = os.spawnvp(os.P_WAIT, l[0], l)
  67     elif isinstance(l, str):
  68         tmp = ['sh', '-c', l]
  69         rc = os.spawnvp(os.P_WAIT, tmp[0], tmp)
  70     if chdir:
  71         os.chdir(cwd)
  72     return rc
  73
  74 def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'):
  75     time.sleep(5)
  76     modules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  77         dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'installed')]
  78     )
  79     if modules and len(modules) > 1:
  80         time.sleep(1)
  81         toinstallmodules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  82             dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'to install')]
  83         )
  84         if toinstallmodules:
  85             print("Package test: FAILED. Not able to install dependencies of base.")
  86             raise Exception("Installation of package failed")
  87         else:
  88             print("Package test: successfuly installed %s modules" % len(modules))
  89     else:
  90         print("Package test: FAILED. Not able to install base.")
  91         raise Exception("Installation of package failed")
  92
  93 def publish(o, releases):
  94     def _publish(o, release):
  95         extension = ''.join(release.split('.', 1)[1])
  96         release_extension = PUBLISH_DIRS[extension][1] if isinstance(PUBLISH_DIRS[extension], list) else extension
  97         release_dir = PUBLISH_DIRS[extension][0] if isinstance(PUBLISH_DIRS[extension], list) else PUBLISH_DIRS[extension]
  98
  99         release_filename = 'odoo_%s-%s.%s' % (version, timestamp, release_extension)
 100         release_path = join(o.pub, release_dir, release_filename)
 101
 102         system('mkdir -p %s' % join(o.pub, release_dir))
 103         shutil.move(join(o.build_dir, release), release_path)
 104
 105         if release_extension == 'deb':
 106             temp_path = tempfile.mkdtemp(suffix='debPackages')
 107             system(['cp', release_path, temp_path])
 108             with open(os.path.join(o.pub, 'deb', 'Packages'), 'w') as out:
 109                 subprocess.call(['dpkg-scanpackages', '.'], stdout=out, cwd=temp_path)
 110             shutil.rmtree(temp_path)
 111
 112         # Latest/symlink handler
 113         release_abspath = abspath(release_path)
 114         latest_abspath = release_abspath.replace(timestamp, 'latest')
 115
 116         if os.path.islink(latest_abspath):
 117             os.unlink(latest_abspath)
 118
 119         os.symlink(release_abspath, latest_abspath)
 120
 121     if isinstance(releases, basestring):
 122         _publish(o, releases)
 123     elif isinstance(releases, list):
 124         for release in releases:
 125             _publish(o, release)
 126
 127 class OdooDocker(object):
 128     def __init__(self):
 129         self.log_file = NamedTemporaryFile(mode='w+b', prefix="bash", suffix=".txt", delete=False)
 130         self.port = 8069  # TODO sle: reliable way to get a free port?
 131         self.prompt_re = '(\r\nroot@|bash-).*# '
 132         self.timeout = 600
 133
 134     def system(self, command):
 135         self.docker.sendline(command)
 136         self.docker.expect(self.prompt_re)
 137
 138     def start(self, docker_image, build_dir, pub_dir):
 139         self.build_dir = build_dir
 140         self.pub_dir = pub_dir
 141
 142         self.docker = pexpect.spawn(
 143             'docker run -v %s:/opt/release -p 127.0.0.1:%s:8069'
 144             ' -t -i %s /bin/bash --noediting' % (self.build_dir, self.port, docker_image),
 145             timeout=self.timeout
 146         )
 147         time.sleep(2)  # let the bash start
 148         self.docker.logfile_read = self.log_file
 149         self.id = subprocess.check_output('docker ps -l -q', shell=True)
 150
 151     def end(self):
 152         try:
 153             _rpc_count_modules(port=str(self.port))
 154         except Exception, e:
 155             print('Exception during docker execution: %s:' % str(e))
 156             print('Error during docker execution: printing the bash output:')
 157             with open(self.log_file.name) as f:
 158                 print '\n'.join(f.readlines())
 159             raise
 160         finally:
 161             self.docker.close()
 162             system('docker rm -f %s' % self.id)
 163             self.log_file.close()
 164             os.remove(self.log_file.name)
 165
 166 @contextmanager
 167 def docker(docker_image, build_dir, pub_dir):
 168     _docker = OdooDocker()
 169     try:
 170         _docker.start(docker_image, build_dir, pub_dir)
 171         try:
 172             yield _docker
 173         except Exception, e:
 174             raise
 175     finally:
 176         _docker.end()
 177
 178 class KVM(object):
 179     def __init__(self, o, image, ssh_key='', login='openerp'):
 180         self.o = o
 181         self.image = image
 182         self.ssh_key = ssh_key
 183         self.login = login
 184
 185     def timeout(self,signum,frame):
 186         print "vm timeout kill",self.pid
 187         os.kill(self.pid,15)
 188
 189     def start(self):
 190         l="kvm -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -drive".split(" ")
 191         #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image)
 192         l.append('file=%s,snapshot=on'%self.image)
 193         #l.extend(['-vnc','127.0.0.1:1'])
 194         l.append('-nographic')
 195         print " ".join(l)
 196         self.pid=os.spawnvp(os.P_NOWAIT, l[0], l)
 197         time.sleep(10)
 198         signal.alarm(2400)
 199         signal.signal(signal.SIGALRM, self.timeout)
 200         try:
 201             self.run()
 202         finally:
 203             signal.signal(signal.SIGALRM, signal.SIG_DFL)
 204             os.kill(self.pid,15)
 205             time.sleep(10)
 206
 207     def ssh(self,cmd):
 208         l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd]
 209         system(l)
 210
 211     def rsync(self,args,options='--delete --exclude .bzrignore'):
 212         cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args)
 213         system(cmd)
 214
 215     def run(self):
 216         pass
 217
 218 class KVMWinBuildExe(KVM):
 219     def run(self):
 220         with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
 221             f.write("VERSION=%s\n" % self.o.version_full)
 222         with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
 223             f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', ''))
 224
 225         self.ssh("mkdir -p build")
 226         self.rsync('%s/ %s@127.0.0.1:build/server/' % (self.o.build_dir, self.login))
 227         self.ssh("cd build/server/setup/win32;time make allinone;")
 228         self.rsync('%s@127.0.0.1:build/server/setup/win32/release/ %s/' % (self.login, self.o.build_dir), '')
 229         print "KVMWinBuildExe.run(): done"
 230
 231 class KVMWinTestExe(KVM):
 232     def run(self):
 233         # Cannot use o.version_full when the version is not correctly parsed
 234         # (for instance, containing *rc* or *dev*)
 235         setuppath = glob("%s/openerp-server-setup-*.exe" % self.o.build_dir)[0]
 236         setupfile = setuppath.split('/')[-1]
 237         setupversion = setupfile.split('openerp-server-setup-')[1].split('.exe')[0]
 238
 239         self.rsync('"%s" %s@127.0.0.1:' % (setuppath, self.login))
 240         self.ssh("TEMP=/tmp ./%s /S" % setupfile)
 241         self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"Odoo %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion)
 242         self.ssh('/cygdrive/c/"Program Files"/"Odoo %s"/server/openerp-server.exe -d mycompany -i base --stop-after-init' % setupversion)
 243         self.ssh('net start odoo-server-8.0')
 244         _rpc_count_modules(port=18069)
 245
 246 #----------------------------------------------------------
 247 # Stage: building
 248 #----------------------------------------------------------
 249 def _prepare_build_dir(o):
 250     cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
 251     system(cmd + ['%s/' % o.odoo_dir, o.build_dir])
 252     for i in glob(join(o.build_dir, 'addons/*')):
 253         shutil.move(i, join(o.build_dir, 'openerp/addons'))
 254
 255 def build_tgz(o):
 256     system(['python2', 'setup.py', '--quiet', 'sdist'], o.build_dir)
 257     system(['cp', glob('%s/dist/openerp-*.tar.gz' % o.build_dir)[0], '%s/odoo.tar.gz' % o.build_dir])
 258
 259 def build_deb(o):
 260     system(['dpkg-buildpackage', '-rfakeroot', '-uc', '-us'], o.build_dir)
 261     system(['cp', glob('%s/../odoo_*.deb' % o.build_dir)[0], '%s/odoo.deb' % o.build_dir])
 262     system(['cp', glob('%s/../odoo_*.dsc' % o.build_dir)[0], '%s/odoo.dsc' % o.build_dir])
 263     system(['cp', glob('%s/../odoo_*_amd64.changes' % o.build_dir)[0], '%s/odoo_amd64.changes' % o.build_dir])
 264     system(['cp', glob('%s/../odoo_*.tar.gz' % o.build_dir)[0], '%s/odoo.deb.tar.gz' % o.build_dir])
 265
 266 def build_rpm(o):
 267     system(['python2', 'setup.py', '--quiet', 'bdist_rpm'], o.build_dir)
 268     system(['cp', glob('%s/dist/openerp-*.noarch.rpm' % o.build_dir)[0], '%s/odoo.noarch.rpm' % o.build_dir])
 269     system(['cp', glob('%s/dist/openerp-*.src.rpm' % o.build_dir)[0], '%s/odoo.src.rpm' % o.build_dir])
 270
 271 def build_exe(o):
 272     KVMWinBuildExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 273     system(['cp', glob('%s/openerp*.exe' % o.build_dir)[0], '%s/odoo.exe' % o.build_dir])
 274
 275 #----------------------------------------------------------
 276 # Stage: testing
 277 #----------------------------------------------------------
 278 def test_tgz(o):
 279     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 280         wheezy.release = 'odoo.tar.gz'
 281         wheezy.system('apt-get update -qq && apt-get upgrade -qq -y')
 282         wheezy.system("apt-get install postgresql python-dev postgresql-server-dev-all python-pip build-essential libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libssl-dev libjpeg-dev -y")
 283         wheezy.system("service postgresql start")
 284         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 285         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 286         wheezy.system('pip install -r /opt/release/requirements.txt')
 287         wheezy.system('/usr/local/bin/pip install /opt/release/%s' % wheezy.release)
 288         wheezy.system("useradd --system --no-create-home odoo")
 289         wheezy.system('su postgres -s /bin/bash -c "createuser -s odoo"')
 290         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 291         wheezy.system('mkdir /var/lib/odoo')
 292         wheezy.system('chown odoo:odoo /var/lib/odoo')
 293         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany -i base --stop-after-init"')
 294         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany &"')
 295
 296 def test_deb(o):
 297     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 298         wheezy.release = 'odoo.deb'
 299         wheezy.system('/usr/bin/apt-get update -qq && /usr/bin/apt-get upgrade -qq -y')
 300         wheezy.system("apt-get install postgresql -y")
 301         wheezy.system("service postgresql start")
 302         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 303         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 304         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 305         wheezy.system('/usr/bin/dpkg -i /opt/release/%s' % wheezy.release)
 306         wheezy.system('/usr/bin/apt-get install -f -y')
 307         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 308         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany &"')
 309
 310 def test_rpm(o):
 311     with docker('centos:centos7', o.build_dir, o.pub) as centos7:
 312         centos7.release = 'odoo.noarch.rpm'
 313         centos7.system('rpm -Uvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-2.noarch.rpm')
 314         centos7.system('yum update -y && yum upgrade -y')
 315         centos7.system('yum install python-pip gcc python-devel -y')
 316         centos7.system('pip install pydot pyPdf vatnumber xlwt http://download.gna.org/pychart/PyChart-1.39.tar.gz')
 317         centos7.system('yum install postgresql postgresql-server postgresql-libs postgresql-contrib postgresql-devel -y')
 318         centos7.system('mkdir -p /var/lib/postgres/data')
 319         centos7.system('chown -R postgres:postgres /var/lib/postgres/data')
 320         centos7.system('chmod 0700 /var/lib/postgres/data')
 321         centos7.system('su postgres -c "initdb -D /var/lib/postgres/data -E UTF-8"')
 322         centos7.system('cp /usr/share/pgsql/postgresql.conf.sample /var/lib/postgres/data/postgresql.conf')
 323         centos7.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
 324         centos7.system('su postgres -c "createdb mycompany"')
 325         centos7.system('export PYTHONPATH=${PYTHONPATH}:/usr/local/lib/python2.7/dist-packages')
 326         centos7.system('su postgres -c "createdb mycompany"')
 327         centos7.system('yum install /opt/release/%s -y' % centos7.release)
 328         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 329         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany &"')
 330
 331 def test_exe(o):
 332     KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 333
 334 #----------------------------------------------------------
 335 # Options and Main
 336 #----------------------------------------------------------
 337 def options():
 338     op = optparse.OptionParser()
 339     root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 340     build_dir = "%s-%s" % (root, timestamp)
 341
 342     op.add_option("-b", "--build-dir", default=build_dir, help="build directory (%default)", metavar="DIR")
 343     op.add_option("-p", "--pub", default=None, help="pub directory (%default)", metavar="DIR")
 344     op.add_option("", "--no-testing", action="store_true", help="don't test the builded packages")
 345     op.add_option("-v", "--version", default='8.0', help="version (%default)")
 346
 347     op.add_option("", "--no-debian", action="store_true", help="don't build the debian package")
 348     op.add_option("", "--no-rpm", action="store_true", help="don't build the rpm package")
 349     op.add_option("", "--no-tarball", action="store_true", help="don't build the tarball")
 350     op.add_option("", "--no-windows", action="store_true", help="don't build the windows package")
 351
 352     # Windows VM
 353     op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/winxp27/winxp27.vdi', help="%default")
 354     op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/winxp27/id_rsa', help="%default")
 355     op.add_option("", "--vm-winxp-login", default='Naresh', help="Windows login (%default)")
 356     op.add_option("", "--vm-winxp-python-version", default='2.7', help="Windows Python version installed in the VM (default: %default)")
 357
 358     (o, args) = op.parse_args()
 359     # derive other options
 360     o.odoo_dir = root
 361     o.pkg = join(o.build_dir, 'pkg')
 362     o.version_full = '%s-%s' % (o.version, timestamp)
 363     o.work = join(o.build_dir, 'openerp-%s' % o.version_full)
 364     o.work_addons = join(o.work, 'openerp', 'addons')
 365     return o
 366
 367 def main():
 368     o = options()
 369     _prepare_build_dir(o)
 370     try:
 371         if not o.no_tarball:
 372             build_tgz(o)
 373             try:
 374                 if not o.no_testing:
 375                     test_tgz(o)
 376                 publish(o, 'odoo.tar.gz')
 377             except Exception, e:
 378                 print("Won't publish the tgz release.\n Exception: %s" % str(e))
 379         if not o.no_debian:
 380             build_deb(o)
 381             try:
 382                 if not o.no_testing:
 383                     test_deb(o)
 384                 publish(o, ['odoo.deb', 'odoo.dsc', 'odoo_amd64.changes', 'odoo.deb.tar.gz'])
 385             except Exception, e:
 386                 print("Won't publish the deb release.\n Exception: %s" % str(e))
 387         if not o.no_rpm:
 388             build_rpm(o)
 389             try:
 390                 if not o.no_testing:
 391                     test_rpm(o)
 392                 publish(o, ['odoo.noarch.rpm', 'odoo.src.rpm'])
 393             except Exception, e:
 394                 print("Won't publish the rpm release.\n Exception: %s" % str(e))
 395         if not o.no_windows:
 396             build_exe(o)
 397             try:
 398                 if not o.no_testing:
 399                     test_exe(o)
 400                 publish(o, 'odoo.exe')
 401             except Exception, e:
 402                 print("Won't publish the exe release.\n Exception: %s" % str(e))
 403     except:
 404         pass
 405     finally:
 406         for leftover in glob('%s/../odoo_*' % o.build_dir):
 407             os.remove(leftover)
 408
 409         shutil.rmtree(o.build_dir)
 410         print('Build dir %s removed' % o.build_dir)
 411
 412         if not o.no_testing:
 413             system("docker rm -f `docker ps -a | awk '{print $1 }'` 2>>/dev/null")
 414             print('Remaining dockers removed')
 415
 416
 417 if __name__ == '__main__':
 418     main()