Skip to content

Caching Remote p2 Repositories

Jump to bottom Edit New page
Lutz Wrage edited this page Jan 12, 2020 · 4 revisions

Introduction

Eclipse provides a mechanism to create a local copy of a (parts of) a remote p2 repository. This is called mirroring. The mirror can be created via an ant task that is executed with the eclipse org.eclipse.ant.core.antRunner application.

Using mirrored repositories has the advantage that the build works even when a p2 repository is not reachable because of server or networking issues. In particular, it works behind a restrictive Internet proxy. In addition, the dependency resolution phase at the beginning of a maven build should be faster, e.g., on one laptop a local build takes 33% less time when using mirrors.

The ant script

This is the build script used to mirror all content from p2 repositories required to build OSATE 2.7.0.

<project name="Create Mirror" default="create-mirror" basedir=".">

  <target name="create-mirror">

    <p2.mirror source="http://download.eclipse.org/releases/2019-12">
      <destination location="${target.dir}/releases/2019-12"/>
      <iu id="org.eclipse.e4.rcp.feature.group"/>
      <iu id="org.eclipse.egit.feature.group"/>
      <iu id="org.eclipse.emf.sdk.feature.group"/>
      <iu id="org.eclipse.emf.transaction.feature.group"/>
      <iu id="org.eclipse.emf.transaction.sdk.feature.group"/>
      <iu id="org.eclipse.equinox.executable.feature.group"/>
      <iu id="org.eclipse.equinox.p2.core.feature.feature.group"/>
      <iu id="org.eclipse.equinox.p2.discovery.feature.feature.group"/>
      <iu id="org.eclipse.equinox.p2.extras.feature.feature.group"/>
      <iu id="org.eclipse.equinox.p2.rcp.feature.feature.group"/>
      <iu id="org.eclipse.equinox.p2.user.ui.feature.group"/>
      <iu id="org.eclipse.gmf.runtime.sdk.feature.group"/>
      <iu id="org.eclipse.graphiti.feature.feature.group"/>
      <iu id="org.eclipse.graphiti.export.feature.feature.group"/>
      <iu id="org.eclipse.graphiti.feature.tools.feature.group"/>
      <iu id="org.eclipse.graphiti.sdk.feature.feature.group"/>
      <iu id="org.eclipse.help.feature.group"/>
      <iu id="org.eclipse.m2e.logback.feature.feature.group"/>
      <iu id="org.eclipse.mylyn.context_feature.feature.group"/>
      <iu id="org.eclipse.platform.feature.group"/>
      <iu id="org.eclipse.rcp.feature.group"/>
      <iu id="org.eclipse.sdk.feature.group"/>
      <iu id="org.eclipse.sirius.common.acceleo.aql"/>
      <iu id="org.eclipse.sirius.runtime.ide.ui.feature.group"/>
      <iu id="org.eclipse.uml2.feature.group"/>
      <iu id="org.eclipse.zest.feature.group"/>
      <iu id="org.eclipse.equinox.ds"/>
      <iu id="org.eclipse.xpand"/>
      <iu id="org.eclipse.xtend"/>
      <iu id="org.eclipse.xtend.typesystem.emf"/>
    </p2.mirror>

    <p2.mirror source="http://download.eclipse.org/birt/update-site/2018-09-interim">
      <destination location="${target.dir}/birt/update-site/2018-09-interim"/>
      <iu id="org.eclipse.birt.chart.engine.extension"/>
      <iu id="org.eclipse.birt.chart.engine"/>
      <iu id="org.eclipse.birt.chart.reportitem"/>
      <iu id="org.eclipse.birt.core"/>
      <iu id="org.eclipse.birt.data.aggregation"/>
      <iu id="org.eclipse.birt.data"/>
      <iu id="org.eclipse.birt.report.data.adapter"/>
      <iu id="org.eclipse.birt.report.engine.emitter.config.docx"/>
      <iu id="org.eclipse.birt.report.engine.emitter.config.excel"/>
      <iu id="org.eclipse.birt.report.engine.emitter.config.html"/>
      <iu id="org.eclipse.birt.report.engine.emitter.config.odp"/>
      <iu id="org.eclipse.birt.report.engine.emitter.config.ods"/>
      <iu id="org.eclipse.birt.report.engine.emitter.config.odt"/>
      <iu id="org.eclipse.birt.report.engine.emitter.config.pdf"/>
      <iu id="org.eclipse.birt.report.engine.emitter.config.pptx"/>
      <iu id="org.eclipse.birt.report.engine.emitter.config"/>
      <iu id="org.eclipse.birt.report.engine.emitter.docx"/>
      <iu id="org.eclipse.birt.report.engine.emitter.html"/>
      <iu id="org.eclipse.birt.report.engine.emitter.odp"/>
      <iu id="org.eclipse.birt.report.engine.emitter.ods"/>
      <iu id="org.eclipse.birt.report.engine.emitter.odt"/>
      <iu id="org.eclipse.birt.report.engine.emitter.pdf"/>
      <iu id="org.eclipse.birt.report.engine.emitter.ppt"/>
      <iu id="org.eclipse.birt.report.engine.emitter.pptx"/>
      <iu id="org.eclipse.birt.report.engine.emitter.wpml"/>
      <iu id="org.eclipse.birt.report.engine.fonts"/>
      <iu id="org.eclipse.birt.report.engine.odf"/>
      <iu id="org.eclipse.birt.report.engine.ooxml"/>
      <iu id="org.eclipse.birt.report.engine.script.javascript"/>
      <iu id="org.eclipse.birt.report.engine"/>
      <iu id="org.eclipse.birt.report.item.crosstab.core"/>
      <iu id="org.eclipse.birt.report.model"/>
      <iu id="uk.co.spudsoft.birt.emitters.excel"/>
    </p2.mirror>

    <p2.mirror source="http://download.eclipse.org/tools/orbit/downloads/drops/R20191126223242/repository">
      <destination location="${target.dir}/tools/orbit/downloads/drops/R20191126223242/repository"/>
      <iu id="org.apache.commons.io" version="2.2.0.v201405211200"/>
    </p2.mirror>

    <p2.mirror source="http://download.eclipse.org/modeling/tmf/xtext/updates/releases/2.20.0">
      <destination location="${target.dir}/modeling/tmf/xtext/updates/releases/2.20.0"/>
      <iu id="org.eclipse.xtext.generator"/>
      <iu id="org.eclipse.xtext.redist.feature.group"/>
      <iu id="org.eclipse.xtext.junit4"/>
      <iu id="org.eclipse.xtext.testing"/>
      <iu id="org.eclipse.xtext.xbase.junit"/>
    </p2.mirror>

    <p2.mirror source="http://download.eclipse.org/modeling/emft/mwe/updates/releases/2.11.1">
      <destination location="${target.dir}/modeling/emft/mwe/updates/releases/2.11.1"/>
      <iu id="org.eclipse.emf.mwe2.launcher.feature.group"/>
    </p2.mirror>

    <p2.mirror source="http://download.eclipse.org/xsemantics/milestones/1.19">
      <destination location="${target.dir}/xsemantics/milestones/1.19"/>
      <iu id="org.eclipse.xsemantics.runtime.feature.feature.group"/>
    </p2.mirror>

    <p2.mirror source="http://download.eclipse.org/elk/updates/releases/0.5.0">
      <destination location="${target.dir}/elk/updates/releases/0.5.0"/>
      <iu id="org.eclipse.elk.feature.feature.group"/>
      <iu id="org.eclipse.elk.algorithms.feature.feature.group"/>
    </p2.mirror>

    <p2.mirror source="http://download.eclipse.org/ease/update/release">
      <destination location="${target.dir}/ease/update/release"/>
      <iu id="org.eclipse.ease.feature.feature.group"/>
      <iu id="org.eclipse.ease.lang.python.py4j.feature.feature.group"/>
      <iu id="org.eclipse.ease.lang.scriptarchive.feature.feature.group"/>
      <iu id="org.eclipse.ease.lang.unittest.feature.feature.group"/>
      <iu id="org.eclipse.ease.modules.feature.feature.group"/>
      <iu id="org.eclipse.ease.ui.feature.feature.group"/>
    </p2.mirror>

    <p2.mirror source="https://eclipse.py4j.org">
      <destination location="${target.dir}/py4j"/>
      <iu id="org.py4j.feature.feature.group"/>
    </p2.mirror>

    <p2.mirror source="https://dl.bintray.com/itemis/generic/xtext-testing/updates/0.2">
      <destination location="${target.dir}/itemis/generic/xtext-testing/updates/0.2"/>
    </p2.mirror>

    <p2.mirror source="http://osate-build.sei.cmu.edu/p2/ocarina">
      <destination location="${target.dir}/ocarina"/>
    </p2.mirror>

    <p2.mirror source="http://osate-build.sei.cmu.edu/download/osate/stable/2.3.7/updates">
      <destination location="${target.dir}/resolute"/>
      <iu id="com.rockwellcollins.atc.resolute.feature.feature.group"/>
    </p2.mirror>

    <p2.mirror source="http://updates.yakindu.com/statecharts/releases/3.5.9">
      <destination location="${target.dir}/yakindu"/>
      <iu id="org.yakindu.base.xtext.utils.jface"/>
    </p2.mirror>

  </target>

</project>

Running the ant script

The ant script requires an eclipse installation to be executed. To run a script mirror.xml execute a command similar to the following with appropriate path names inserted.

<path to eclipse>/eclipse \
    -application org.eclipse.ant.core.antRunner \
    -buildfile mirror.xml \
    -Dtarget.dir=<path to target directory>/p2mirror

The result is a number of local p2 repositories. Note that this takes a while.

An archive of the p2mirror directory for building OSATE is available in a google drive folder.

If you want to use the mirrored repositories on a build server, create a mirrors on your desktop or laptop and copy the mirrors to the server.

Using the mirrored repositories

Maven can redirect repository locations by adding mirror elements to the local settings.xml file. On Linux this file is by default in $HOME/.m2. For example:

<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
    <profiles>
        <profile>
            <id>allow-snapshots</id>
            <activation>
                <activeByDefault>true</activeByDefault>
            </activation>
            <repositories>
                <repository>
                    <id>snapshots-repo</id>
                    <url>https://oss.sonatype.org/content/repositories/snapshots</url>
                    <releases>
                        <enabled>false</enabled>
                    </releases>
                    <snapshots>
                        <enabled>true</enabled>
                    </snapshots>
                </repository>
            </repositories>
            <pluginRepositories>
                <pluginRepository>
                    <id>plugin-snapshots-repo</id>
                    <url>https://oss.sonatype.org/content/repositories/snapshots</url>
                    <releases>
                        <enabled>false</enabled>
                    </releases>
                    <snapshots>
                        <enabled>true</enabled>
                    </snapshots>
                </pluginRepository>
            </pluginRepositories>
        </profile>
    </profiles>

    <mirrors>
        <mirror>
            <id>eclipse-mirror</id>
            <mirrorOf>http://download.eclipse.org</mirrorOf>
            <url>file:/home/lw/tmp/mirrortest/p2mirror</url>
            <layout>p2</layout>
            <mirrorOfLayouts>p2</mirrorOfLayouts>
        </mirror>
        <mirror>
            <id>py4j-mirror</id>
            <mirrorOf>http://eclipse.py4j.org</mirrorOf>
            <url>file:/home/lw/tmp/mirrortest/p2mirror/py4j</url>
            <layout>p2</layout>
            <mirrorOfLayouts>p2</mirrorOfLayouts>
        </mirror>
        <mirror>
            <id>itemis-mirror</id>
            <mirrorOf>https://dl.bintray.com/itemis</mirrorOf>
            <url>file:/home/lw/tmp/mirrortest/p2mirror/itemis</url>
            <layout>p2</layout>
            <mirrorOfLayouts>p2</mirrorOfLayouts>
        </mirror>
        <mirror>
            <id>ocarina-mirror</id>
            <mirrorOf>http://osate-build.sei.cmu.edu/p2/ocarina</mirrorOf>
            <url>file:/home/lw/tmp/mirrortest/p2mirror/ocarina</url>
            <layout>p2</layout>
            <mirrorOfLayouts>p2</mirrorOfLayouts>
        </mirror>
        <mirror>
            <id>resolute-mirror</id>
            <mirrorOf>http://osate-build.sei.cmu.edu/download/osate/stable/2.3.7/updates</mirrorOf>
            <url>file:/home/lw/tmp/mirrortest/p2mirror/resolute</url>
            <layout>p2</layout>
            <mirrorOfLayouts>p2</mirrorOfLayouts>
        </mirror>
        <mirror>
            <id>yakindu-mirror</id>
            <mirrorOf>http://updates.yakindu.com/statecharts/releases</mirrorOf>
            <url>file:/home/lw/tmp/mirrortest/p2mirror/yakindu</url>
            <layout>p2</layout>
            <mirrorOfLayouts>p2</mirrorOfLayouts>
        </mirror>
    </mirrors>
</settings>

References

  1. Eclipse Help
  2. Eclipse wiki
  3. Blog posts by Lorenzo Bettini
Add a custom footer
Add a custom sidebar

Clone this wiki locally