Merge pull request #1432 from LuaAndC/skeleton

add tool skeleton.py
2016-07-14 17:51:31 +10:00 · 2016-07-14 17:51:31 +10:00 · 837e5e705c
parent de9611cfee 38a5c97d0b
commit 837e5e705c
3 changed files with 354 additions and 55 deletions
--- a/15
+++ b/15
@ -369,16 +369,18 @@ $(PREFIX)/installed/print-git-oneline-$(GIT_HEAD): | $(PREFIX)/installed/.gitkee
 	@rm -f '$(PREFIX)/installed/print-git-oneline-'*
 	@touch '$@'

-# include core MXE packages and set *_MAKEFILE
+# include core MXE packages and set base filenames
 include $(patsubst %,$(TOP_DIR)/src/%.mk,$(PKGS))
 $(foreach PKG,$(PKGS),\
-    $(eval $(PKG)_MAKEFILE := $(realpath $(TOP_DIR)/src/$(PKG).mk)))
+    $(eval $(PKG)_MAKEFILE  := $(realpath $(TOP_DIR)/src/$(PKG).mk)) \
+    $(eval $(PKG)_TEST_FILE := $(realpath $(wildcard $(TOP_DIR)/src/$(PKG)-test.*))))

-# include files from MXE_PLUGIN_DIRS, set *_MAKEFILE and `all-<plugin>` target
+# include files from MXE_PLUGIN_DIRS, set base filenames and `all-<plugin>` target
 PLUGIN_FILES := $(realpath $(wildcard $(addsuffix /*.mk,$(MXE_PLUGIN_DIRS))))
 PLUGIN_PKGS  := $(basename $(notdir $(PLUGIN_FILES)))
 $(foreach FILE,$(PLUGIN_FILES),\
-    $(eval $(basename $(notdir $(FILE)))_MAKEFILE ?= $(FILE)) \
+    $(eval $(basename $(notdir $(FILE)))_MAKEFILE  ?= $(FILE)) \
+    $(eval $(basename $(notdir $(FILE)))_TEST_FILE ?= $(wildcard $(basename $(FILE))-test.*)) \
    $(eval all-$(lastword $(call split,/,$(dir $(FILE)))): $(basename $(notdir $(FILE)))))
 include $(PLUGIN_FILES)
 PKGS := $(sort $(PKGS) $(PLUGIN_PKGS))
@ -525,6 +527,9 @@ build-only-$(1)_$(3): TARGET = $(3)
 build-only-$(1)_$(3): BUILD_$(if $(findstring shared,$(3)),SHARED,STATIC) = TRUE
 build-only-$(1)_$(3): LIB_SUFFIX = $(if $(findstring shared,$(3)),dll,a)
 build-only-$(1)_$(3): BITS = $(if $(findstring x86_64,$(3)),64,32)
+build-only-$(1)_$(3): SOURCE_DIR = $(2)/$($(1)_SUBDIR)
+build-only-$(1)_$(3): BUILD_DIR  = $(2)/$($(1)_SUBDIR).build_
+build-only-$(1)_$(3): TEST_FILE  = $($(1)_TEST_FILE)
 build-only-$(1)_$(3): CMAKE_RUNRESULT_FILE = $(PREFIX)/share/cmake/modules/TryRunResults.cmake
 build-only-$(1)_$(3): CMAKE_TOOLCHAIN_FILE = $(PREFIX)/$(3)/share/cmake/mxe-conf.cmake
 build-only-$(1)_$(3): CMAKE_TOOLCHAIN_DIR  = $(PREFIX)/$(3)/share/cmake/mxe-conf.d
@ -543,6 +548,8 @@ build-only-$(1)_$(3):
 	    perl --version 2>&1 | head -3
 	    rm -rf   '$(2)'
 	    mkdir -p '$(2)'
+	    mkdir -p '$$(SOURCE_DIR)'
+	    mkdir -p '$$(BUILD_DIR)'

 	    # disable wine with readonly directory
 	    # see https://github.com/mxe/mxe/issues/841
--- a/index.html
+++ b/index.html
@ -1141,10 +1141,6 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">aubio</td>
        <td class="website"><a href="http://www.aubio.org/">aubio</a></td>
    </tr>
-    <tr>
-        <td class="package">db</td>
-        <td class="website"><a href="http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/overview/index.html">Oracle Berkeley DB</a></td>
-    </tr>
    <tr>
        <td class="package">bfd</td>
        <td class="website"><a href="https://www.gnu.org/software/binutils/">Binary File Descriptor library</a></td>
@ -1213,14 +1209,14 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">chromaprint</td>
        <td class="website"><a href="http://acoustid.org/chromaprint">Chromaprint</a></td>
    </tr>
-    <tr>
-        <td class="package">cloog</td>
-        <td class="website"><a href="http://www.cloog.org/">CLooG Code Generator</a></td>
-    </tr>
    <tr>
        <td class="package">cimg</td>
        <td class="website"><a href="http://cimg.eu/">CImg Library</a></td>
    </tr>
+    <tr>
+        <td class="package">cloog</td>
+        <td class="website"><a href="http://www.cloog.org/">CLooG Code Generator</a></td>
+    </tr>
    <tr>
        <td class="package">cmake</td>
        <td class="website"><a href="http://www.cmake.org/">cmake</a></td>
@ -1261,6 +1257,10 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">curl</td>
        <td class="website"><a href="http://curl.haxx.se/libcurl/">cURL</a></td>
    </tr>
+    <tr>
+        <td class="package">db</td>
+        <td class="website"><a href="http://www.oracle.com/technetwork/database/database-technologies/berkeleydb/overview/index.html">Oracle Berkeley DB</a></td>
+    </tr>
    <tr>
        <td class="package">dbus</td>
        <td class="website"><a href="http://dbus.freedesktop.org/">dbus</a></td>
@ -1865,14 +1865,14 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">libpng</td>
        <td class="website"><a href="http://www.libpng.org/">libpng</a></td>
    </tr>
-    <tr>
-        <td class="package">librsvg</td>
-        <td class="website"><a href="http://librsvg.sourceforge.net/">librsvg</a></td>
-    </tr>
    <tr>
        <td class="package">librosco</td>
        <td class="website"><a href="https://github.com/colinbourassa/librosco/">librosco</a></td>
    </tr>
+    <tr>
+        <td class="package">librsvg</td>
+        <td class="website"><a href="http://librsvg.sourceforge.net/">librsvg</a></td>
+    </tr>
    <tr>
        <td class="package">librtmp</td>
        <td class="website"><a href="http://rtmpdump.mplayerhq.hu/">librtmp</a></td>
@ -1997,6 +1997,10 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">mdbtools</td>
        <td class="website"><a href="http://sourceforge.net/projects/mdbtools/">mdbtools</a></td>
    </tr>
+    <tr>
+        <td class="package">mingw-w64</td>
+        <td class="website"><a href="http://mingw-w64.sourceforge.net/">MinGW-w64 Runtime</a></td>
+    </tr>
    <tr>
        <td class="package">miniupnpc</td>
        <td class="website"><a href="http://miniupnp.free.fr/">miniupnpc</a></td>
@ -2005,10 +2009,6 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">minizip</td>
        <td class="website"><a href="http://www.winimage.com/zLibDll/minizip.html">minizip</a></td>
    </tr>
-    <tr>
-        <td class="package">mingw-w64</td>
-        <td class="website"><a href="http://mingw-w64.sourceforge.net/">MinGW-w64 Runtime</a></td>
-    </tr>
    <tr>
        <td class="package">mman-win32</td>
        <td class="website"><a href="https://code.google.com/p/mman-win32/">MMA-Win32</a></td>
@ -2150,7 +2150,7 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="website"><a href="https://www.openssl.org/">openssl</a></td>
    </tr>
    <tr>
-      <td class="package">openthreads</td>
+        <td class="package">openthreads</td>
        <td class="website"><a href="http://www.openscenegraph.org/">OpenThreads</a></td>
    </tr>
    <tr>
@ -2374,11 +2374,11 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="website"><a href="http://qt-project.org/">Qt</a></td>
    </tr>
    <tr>
-        <td class="package">qtscxml</td>
+        <td class="package">qtscript</td>
        <td class="website"><a href="http://qt-project.org/">Qt</a></td>
    </tr>
    <tr>
-        <td class="package">qtscript</td>
+        <td class="package">qtscxml</td>
        <td class="website"><a href="http://qt-project.org/">Qt</a></td>
    </tr>
    <tr>
@ -2485,6 +2485,30 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">sdl</td>
        <td class="website"><a href="http://www.libsdl.org/">SDL</a></td>
    </tr>
+    <tr>
+        <td class="package">sdl2</td>
+        <td class="website"><a href="http://www.libsdl.org/">SDL2</a></td>
+    </tr>
+    <tr>
+        <td class="package">sdl2_gfx</td>
+        <td class="website"><a href="http://www.ferzkopp.net/joomla/content/view/19/14/">SDL2_gfx</a></td>
+    </tr>
+    <tr>
+        <td class="package">sdl2_image</td>
+        <td class="website"><a href="http://www.libsdl.org/">SDL2_image</a></td>
+    </tr>
+    <tr>
+        <td class="package">sdl2_mixer</td>
+        <td class="website"><a href="http://www.libsdl.org/">SDL2_mixer</a></td>
+    </tr>
+    <tr>
+        <td class="package">sdl2_net</td>
+        <td class="website"><a href="http://www.libsdl.org/">sdl2_net</a></td>
+    </tr>
+    <tr>
+        <td class="package">sdl2_ttf</td>
+        <td class="website"><a href="http://www.libsdl.org/">SDL2_ttf</a></td>
+    </tr>
    <tr>
        <td class="package">sdl_gfx</td>
        <td class="website"><a href="http://www.ferzkopp.net/joomla/content/view/19/14/">SDL_gfx</a></td>
@ -2517,30 +2541,6 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">sdl_ttf</td>
        <td class="website"><a href="http://www.libsdl.org/projects/SDL_ttf/">SDL_ttf</a></td>
    </tr>
-    <tr>
-        <td class="package">sdl2</td>
-        <td class="website"><a href="http://www.libsdl.org/">SDL2</a></td>
-    </tr>
-    <tr>
-        <td class="package">sdl2_gfx</td>
-        <td class="website"><a href="http://www.ferzkopp.net/joomla/content/view/19/14/">SDL2_gfx</a></td>
-    </tr>
-    <tr>
-        <td class="package">sdl2_image</td>
-        <td class="website"><a href="http://www.libsdl.org/">SDL2_image</a></td>
-    </tr>
-    <tr>
-        <td class="package">sdl2_mixer</td>
-        <td class="website"><a href="http://www.libsdl.org/">SDL2_mixer</a></td>
-    </tr>
-    <tr>
-        <td class="package">sdl2_net</td>
-        <td class="website"><a href="http://www.libsdl.org/">sdl2_net</a></td>
-    </tr>
-    <tr>
-        <td class="package">sdl2_ttf</td>
-        <td class="website"><a href="http://www.libsdl.org/">SDL2_ttf</a></td>
-    </tr>
    <tr>
        <td class="package">sfml</td>
        <td class="website"><a href="http://www.sfml-dev.org/">SFML</a></td>
@ -2573,14 +2573,14 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">sqlite</td>
        <td class="website"><a href="http://www.sqlite.org/">SQLite</a></td>
    </tr>
-    <tr>
-        <td class="package">suitesparse</td>
-        <td class="website"><a href="http://www.cise.ufl.edu/research/sparse/SuiteSparse/">SuiteSparse</a></td>
-    </tr>
    <tr>
        <td class="package">subversion</td>
        <td class="website"><a href="http://subversion.apache.org">subversion</a></td>
    </tr>
+    <tr>
+        <td class="package">suitesparse</td>
+        <td class="website"><a href="http://www.cise.ufl.edu/research/sparse/SuiteSparse/">SuiteSparse</a></td>
+    </tr>
    <tr>
        <td class="package">t4k_common</td>
        <td class="website"><a href="http://tux4kids.alioth.debian.org/">t4k_common</a></td>
@ -2733,6 +2733,10 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">xvidcore</td>
        <td class="website"><a href="http://www.xvid.org/">xvidcore</a></td>
    </tr>
+    <tr>
+        <td class="package">xxhash</td>
+        <td class="website"><a href="http://cyan4973.github.io/xxHash/">xxHash</a></td>
+    </tr>
    <tr>
        <td class="package">xz</td>
        <td class="website"><a href="http://tukaani.org/xz/">XZ</a></td>
@ -2749,10 +2753,6 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        <td class="package">zziplib</td>
        <td class="website"><a href="http://zziplib.sourceforge.net/">ZZIPlib</a></td>
    </tr>
-    <tr>
-        <td class="package">xxhash</td>
-        <td class="website"><a href="http://cyan4973.github.io/xxHash/">xxHash</a></td>
-    </tr>
    </table>

    <script>
@ -2913,6 +2913,15 @@ local-pkg-list: $(LOCAL_PKG_LIST)</pre>
        The <a href="doc/gmsl.html">GNU Make Standard Library</a> is also
        available (though it should be unnecessary for most packages).
        </p>
+
+        <p>
+        Alternatively you can use tool <code>tools/skeleton.py</code> to
+        create a skeleton of new MXE package. It fills most of the fields
+        of <code>.mk</code> file automatically and supports typical
+        build scenarios through option <code>--builder</code>. It also
+        adds a package to the <a href="#packages">list of packages</a>
+        (see below).
+        </p>
    </li>

    <li>
--- a/tools/skeleton.py
+++ b/tools/skeleton.py
@ -0,0 +1,283 @@
+#!/usr/bin/env python
+
+""" Create a skeleton of new MXE package.
+
+This file is part of MXE.
+See index.html for further information.
+"""
+
+import argparse
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+try:
+    import urllib2
+except:
+    # Python 3
+    import urllib.request as urllib2
+
+MK_TEMPLATE = r'''
+# This file is part of MXE.
+# See index.html for further information.
+
+PKG             := %(name)s
+$(PKG)_IGNORE   :=
+$(PKG)_VERSION  := %(version)s
+$(PKG)_CHECKSUM := %(checksum)s
+$(PKG)_SUBDIR   := %(subdir_template)s
+$(PKG)_FILE     := %(filename_template)s
+$(PKG)_URL      := %(file_url_template)s
+$(PKG)_DEPS     := gcc
+
+define $(PKG)_UPDATE
+    echo 'TODO: write update script for %(name)s.' >&2;
+    echo $(%(name)s_VERSION)
+endef
+
+define $(PKG)_BUILD
+    %(build)s
+endef
+'''
+
+CMAKE_BUILD = r'''
+    # build and install the library
+    cd '$(BUILD_DIR)' && $(TARGET)-cmake \
+        '$(SOURCE_DIR)'
+    $(MAKE) -C '$(BUILD_DIR)' -j '$(JOBS)'
+    $(MAKE) -C '$(BUILD_DIR)' -j 1 install
+'''
+
+AUTOTOOLS_BUILD = r'''
+    # build and install the library
+    cd '$(BUILD_DIR)' && $(SOURCE_DIR)/configure \
+        $(MXE_CONFIGURE_OPTS)
+    $(MAKE) -C '$(BUILD_DIR)' -j '$(JOBS)'
+    $(MAKE) -C '$(BUILD_DIR)' -j 1 install \
+        bin_PROGRAMS= \
+        sbin_PROGRAMS= \
+        noinst_PROGRAMS=
+'''
+
+MAKE_BUILD = r'''
+    # build and install the library
+    $(MAKE) -C '$(SOURCE_DIR)' -j '$(JOBS)'
+    $(MAKE) -C '$(SOURCE_DIR)' -j 1 install
+'''
+
+BUILDERS = {
+    'autotools': AUTOTOOLS_BUILD,
+    'make': MAKE_BUILD,
+    'cmake': CMAKE_BUILD,
+}
+
+PC_AND_TEST = r'''
+    # create pkg-config files
+    $(INSTALL) -d '$(PREFIX)/$(TARGET)/lib/pkgconfig'
+    (echo 'Name: $(PKG)'; \
+     echo 'Version: $($(PKG)_VERSION)'; \
+     echo 'Description: %(description)s'; \
+     echo 'Libs: -l%(libname)s';) \
+     > '$(PREFIX)/$(TARGET)/lib/pkgconfig/$(PKG).pc'
+
+    # compile test
+    '$(TARGET)-gcc' \
+        -W -Wall -Werror -ansi -pedantic \
+        '$(TEST_FILE)' -o '$(PREFIX)/$(TARGET)/bin/test-$(PKG).exe' \
+        `'$(TARGET)-pkg-config' $(PKG) --cflags --libs`
+'''
+
+def get_filename(file_url):
+    return file_url.rsplit('/', 1)[1]
+
+def deduce_version(file_url):
+    filename = get_filename(file_url)
+    return re.search(r'\d[\d.-_]+\d|\d', filename).group()
+
+def deduce_website(file_url):
+    return file_url.split('://', 1)[1].split('/', 1)[0]
+
+def download_file(destination, url):
+    with open(destination, 'wb') as d:
+        request =  urllib2.urlopen(url)
+        shutil.copyfileobj(request, d)
+        request.close()
+
+def make_checksum(filepath):
+    hasher = hashlib.sha256()
+    with open(filepath, 'rb') as f:
+        for chunk in iter(lambda: f.read(1024 ** 2), b''):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+def deduce_subdir(archive):
+    args = ['tar', '-tf', archive]
+    tar = subprocess.Popen(args, stdout=subprocess.PIPE)
+    output = tar.communicate()[0]
+    if not isinstance(output, str):
+        # Python 3
+        output = output.decode()
+    files = output.strip().split('\n')
+    first_file = files[0].strip()
+    directory = first_file.split('/', 1)[0]
+    return directory
+
+def make_build(options, builder):
+    commands_template = BUILDERS[builder].lstrip() + PC_AND_TEST.rstrip()
+    return commands_template % options
+
+def update_index_html(name, description, website):
+    # read HTML and find a list of packages
+    with open('index.html', 'rb') as f:
+        index_html = f.read()
+        if not isinstance(index_html, str):
+            # Python 3
+            index_html = index_html.decode()
+    sep1 = '    <table id="package-list" class="old">'
+    sep2 = '    </table>'
+    (prefix, other) = index_html.split(sep1, 1)
+    (packages_html, suffix) = other.split(sep2, 1)
+    # find existing packages
+    pkg_re = r'''
+    <tr>
+        <td class="package">(?P<name>.*)</td>
+        <td class="website"><a href="(?P<website>.*)">(?P<description>.*)</a></td>
+    </tr>
+    '''.strip()
+    packages = [
+        {
+            'name': match.group('name'),
+            'description': match.group('description'),
+            'website': match.group('website'),
+        }
+        for match in re.finditer(pkg_re, packages_html)
+    ]
+    packages.append({
+        'name': name,
+        'description': description,
+        'website': website,
+    })
+    packages.sort(key=lambda package: package['name'])
+    pkg_template = r'''
+    <tr>
+        <td class="package">%(name)s</td>
+        <td class="website"><a href="%(website)s">%(description)s</a></td>
+    </tr>
+    '''.rstrip()
+    packages_html = ''.join(pkg_template % package for package in packages)
+    packages_html += '\n'
+    # build and write HTML
+    index_html = prefix + sep1 + packages_html + sep2 + suffix
+    (_, tmp_index_html) = tempfile.mkstemp()
+    with open(tmp_index_html, 'wt') as f:
+        f.write(index_html)
+    os.rename(tmp_index_html, 'index.html')
+
+def make_skeleton(
+    name,
+    description,
+    file_url,
+    version,
+    subdir,
+    website,
+    builder,
+):
+    mk_filename = 'src/%s.mk' % name
+    if os.path.isfile(mk_filename):
+        raise Exception('File %s exists!' % mk_filename)
+    if description is None:
+        description = name
+    if version is None:
+        version = deduce_version(file_url)
+    if website is None:
+        website = deduce_website(file_url)
+    with tempfile.NamedTemporaryFile() as pkg_file:
+        download_file(pkg_file.name, file_url)
+        checksum = make_checksum(pkg_file.name)
+        if subdir is None:
+            subdir = deduce_subdir(pkg_file.name)
+    filename = get_filename(file_url)
+    filename_template = filename.replace(version, '$($(PKG)_VERSION)')
+    file_url_template = file_url.replace(version, '$($(PKG)_VERSION)')
+    subdir_template = subdir.replace(version, '$($(PKG)_VERSION)')
+    libname = name
+    if libname.startswith('lib'):
+        libname = libname[3:]
+    with open(mk_filename, 'wt') as mk:
+        options = {
+            'name': name,
+            'description': description,
+            'libname': libname,
+            'website': website,
+            'file_url_template': file_url_template,
+            'checksum': checksum,
+            'version': version,
+            'subdir_template': subdir_template,
+            'filename_template': filename_template,
+        }
+        options['build'] = make_build(options, builder)
+        mk.write(MK_TEMPLATE.lstrip() % options)
+    update_index_html(name, description, website)
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        '--name',
+        type=str,
+        help='Package name',
+        required=True,
+    )
+    parser.add_argument(
+        '--file-url',
+        type=str,
+        help='URL with file of package',
+        required=True,
+    )
+    parser.add_argument(
+        '--description',
+        type=str,
+        help='Package description (defaults to name)',
+        required=False,
+    )
+    parser.add_argument(
+        '--version',
+        type=str,
+        help='Package version (can be deduced from file)',
+        required=False,
+    )
+    parser.add_argument(
+        '--subdir',
+        type=str,
+        help='Package subdir (can be deduced from file)',
+        required=False,
+    )
+    parser.add_argument(
+        '--website',
+        type=str,
+        help='Package website (defaults to domain of file)',
+        required=False,
+    )
+    parser.add_argument(
+        '--builder',
+        choices=sorted(BUILDERS.keys()),
+        help='Template of $(PKG)_BUILD',
+        default='autotools',
+    )
+    args = parser.parse_args()
+    make_skeleton(
+        args.name,
+        args.description,
+        args.file_url,
+        args.version,
+        args.subdir,
+        args.website,
+        args.builder,
+    )
+
+if __name__ == '__main__':
+    main()