Add support for --abidw to reduce unnecessary dynamic relinks

RedBeard0531 · Nov 3, 2017 · f19916b · f19916b
1 parent 5f4a8dd
commit f19916b
Show file tree

Hide file tree

Showing 3 changed files with 87 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -50,6 +50,7 @@ up with `--help` so they are documented here.
 
 | Flag | Default | Description |
 | ---- | ------- | ----------- |
+| `--abidw` | off | **EXPERIMENTAL** Use [abidw](#abidw-support) to reduce dynamic relinking when the ABI is unchanged |
 | `--icecream` | off | **LINUX ONLY** Use [icecream](#-icecream-support) for distributed compilation |
 | `--link-pool-depth=NNN` | 4 | **WINDOWS ONLY**: limit the number of concurrent link tasks |
 | `--ninja-builddir=path` | current directory | Where ninja stores [its database](https://ninja-build.org/manual.html#ref_log). **Delete your `build/` directory if you change this!** |
@@ -187,6 +188,24 @@ bottleneck limiting the speed that you can submit jobs to the cluster. You can
 set the `CCACHE_DISABLE=1` environment variable when running ninja to speed up
 your builds with the trade-off that it won't cache the compilations.
 
+## ABIDW support
+
+If you add `--abidb` to your scons flags, the build will use
+[abidw](https://sourceware.org/libabigail/manual/abidw.html) from
+[libabigail](https://sourceware.org/libabigail/) to generate ABI signatures for
+dynamic libraries, and use the signatures to decide if downstream libraries and
+executables need to be relinked. This can result in substantially shorter builds
+when editing a single cpp file (unfortunately header changes don't benefit).
+
+You will need a version of `abidw` that supports the `--no-show-locs` flag which
+was added very recently. This means you will probably need to install
+`libabigail` from git rather that using a package. Except on
+[Arch](https://aur.archlinux.org/packages/libabigail-git/).
+
+While we believe this is safe, it hasn't seen much battle testing on our
+code-base. If you suspect an issue, turn this flag off and let me know. Toggling
+this flag should only require a relink, no recompiling.
+
 ### Installing icecream on Ubuntu (and similar distros)
 
 1. `apt-get install icecc`

diff --git a/build.py b/build.py
@@ -45,9 +45,16 @@
         dest='icecream',
         help='Use the icecream distributed compile server')
 
+AddOption('--abidw',
+        default=False,
+        action='store_true',
+        dest='abidw',
+        help='EXPERIMENTAL Use abidw to reduce dynamic relinking when the ABI is unchanged')
+
 split_lines_script = os.path.join(my_dir, 'split_lines.py')
 subst_file_script = os.path.join(my_dir, 'subst_file.py')
 test_list_script = os.path.join(my_dir, 'test_list.py')
+update_if_changed_script = os.path.join(my_dir, 'update_if_changed.py')
 touch_compiler_timestamps_script = os.path.join(my_dir, 'touch_compiler_timestamps.py')
 
 icecc_create_env = os.path.join(my_dir, 'icecream', 'icecc-create-env')
@@ -103,6 +110,8 @@ def __init__(self, name, env):
         self.add_run_test_builds()
         self.set_up_complier_upgrade_check()
 
+        if GetOption('abidw'):
+            self.set_up_abidw()
         if env.get('_NINJA_USE_ERRCODE'):
             self.add_error_code_check()
         if env.get('_NINJA_CCACHE'):
@@ -157,6 +166,26 @@ def set_up_complier_upgrade_check(self):
             if build['rule'] in ('CC', 'CXX', 'SHCC', 'SHCXX'):
                 build.setdefault('implicit', []).append(self.compiler_timestamp_file)
 
+    def set_up_abidw(self):
+        self.tool_commands['ABIDW'] = ('abidw --no-show-locs $in | md5sum | $PYTHON %s $out'
+                                       % update_if_changed_script)
+
+        shlibsuffix = self.globalEnv['SHLIBSUFFIX']
+        for build in self.builds:
+            if build['rule'] not in ('LINK', 'SHLINK'):
+                continue
+
+            # Generate a lib.abidw signature file for each lib
+            if build['outputs'].endswith(shlibsuffix):
+                self.builds.append(dict(rule='ABIDW',
+                                        inputs=build['outputs'],
+                                        outputs=build['outputs'] + '.abidw'))
+
+            # Replace all lib input edges (to link tasks) with lib.abidw files.
+            for i, f in enumerate(build['implicit']):
+                if f.endswith(shlibsuffix):
+                   build['implicit'][i] = f + '.abidw'
+
     def add_error_code_check(self):
         timestamp_file = os.path.join('build', 'compiler_timestamps', 'error_code_check.timestamp')
         command = self.make_command(
@@ -751,6 +780,12 @@ def write_rules(self, ninja):
                     command = '%s -MMD -MF $out.d'%(self.tool_commands['SHCC']),
                     pool=compile_pool,
                     description = 'SHCC $out')
+            if 'ABIDW' in self.tool_commands:
+                ninja.rule('ABIDW',
+                    command = self.tool_commands['ABIDW'],
+                    pool = local_pool,
+                    restat = 1,
+                    description = 'ABIDW $in')
             if 'SHLINK' in self.tool_commands:
                 command = self.tool_commands['SHLINK']
                 i = command.find('$SHLINK ') + len('$SHLINK')
@@ -767,8 +802,12 @@ def write_rules(self, ninja):
                 i = command.find('$LINK ') + len('$LINK')
                 prefix = command[:i]
                 args = command[i + 1:]
+                suffix = ''
+                if GetOption('abidw'):
+                    # This hack ensures that we relink binaries when switching between using abidw and not.
+                    suffix = ' # using abidw'
                 ninja.rule('LINK',
-                    command = prefix + ' @$out.rsp',
+                    command = prefix + ' @$out.rsp' + suffix,
                     rspfile = '$out.rsp',
                     rspfile_content = args,
                     pool=local_pool,
@@ -899,6 +938,10 @@ def configure(conf, env):
         print "*** ccache is used automatically if it is installed."
         Exit(1)
 
+    if env.get('ABIDW'):
+        print "*** Use --abidw rather than setting the ABIDW variable"
+        Exit(1)
+
     env['NINJA'] = where_is(env, 'ninja')
     if not env['NINJA']:
         env['NINJA'] = where_is(env, 'ninja-build') # Fedora...
@@ -994,6 +1037,16 @@ def configure(conf, env):
                 # are using the same machine.
                 env['_NINJA_ICECC'] = env['_NINJA_ICERUN']
 
+        if GetOption('abidw'):
+            try:
+                subprocess.check_output(['abidw', '--no-show-locs', '/bin/true'],
+                                        stderr=subprocess.STDOUT)
+            except Exception:
+                # Errors checking for updates shouldn't prevent building.
+                print "*** Error running abidw --no-show-locs. This requires a version from the"
+                print "*** libabigail git master branch from at least 2017-11-02."
+                Exit(1)
+
     for ninja_file in ninja_files:
         cmd = env.Command(ninja_file, [], Action(makeNinjaFile, action_str))
         env.Precious(cmd) # Don't delete the .ninja file before building.

diff --git a/update_if_changed.py b/update_if_changed.py
@@ -0,0 +1,14 @@
+import sys
+import os
+
+in_file = sys.stdin.read()
+out_file_name = sys.argv[1]
+
+if os.path.exists(out_file_name):
+    with open(out_file_name, 'r') as f:
+        if f.read() == in_file:
+            sys.exit(0)
+
+with open(out_file_name, 'w') as f:
+    f.write(in_file)
+