[New] Added new MigrationsHandler to migrate the model

- Add the new MigrationsHandler as a StrolchComponent
- add migrations/data/<realm>/n.n.n.xml files
- and your migration will run
This commit is contained in:
Robert von Burg 2015-02-08 12:59:21 +01:00
parent e741680959
commit 35c35b9d1c
25 changed files with 319 additions and 142 deletions

@ -1 +1 @@
Subproject commit d87517e4c2a3e9507b2661cc1737031761d99b61
Subproject commit 8861db85091eea43f06ed2a480360ba670d3c30e

View File

@ -1,20 +0,0 @@
package li.strolch.migrations;
import java.io.File;
import li.strolch.agent.api.ComponentContainer;
import ch.eitchnet.privilege.model.Certificate;
import ch.eitchnet.utils.Version;
public class DataMigration extends Migration {
public DataMigration(String realm, Version version, File dataFile) {
super(realm, version, dataFile);
}
@Override
public void migrate(ComponentContainer container, Certificate certificate) {
logger.info("[" + this.realm + "] Running migration " + this.version);
}
}

View File

@ -1,40 +0,0 @@
package li.strolch.migrations;
import java.io.File;
import li.strolch.agent.api.ComponentContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ch.eitchnet.privilege.model.Certificate;
import ch.eitchnet.utils.Version;
public abstract class Migration {
protected static final Logger logger = LoggerFactory.getLogger(CodeMigration.class);
protected final String realm;
protected final Version version;
protected final File dataFile;
public Migration(String realm, Version version, File dataFile) {
this.realm = realm;
this.version = version;
this.dataFile = dataFile;
}
public String getRealm() {
return realm;
}
public Version getVersion() {
return version;
}
public File getDataFile() {
return dataFile;
}
public abstract void migrate(ComponentContainer container, Certificate certificate);
}

View File

@ -559,10 +559,11 @@ public abstract class AbstractTransaction implements StrolchTransaction {
try {
this.txResult.setState(TransactionState.CLOSING);
// TODO re-think this.
if (!this.commands.isEmpty()) {
String msg = "Current close strategy {0} is readonly and thus does not support doing commands!";
msg = MessageFormat.format(msg, this.closeStrategy);
throw fail(msg);
logger.error("There are commands registered on a read-only transaction. Changing to rollback! Probably due to an exception!");
autoCloseableRollback();
return;
}
long auditTrailDuration = writeAuditTrail();

View File

@ -1,47 +0,0 @@
package li.strolch.migrations;
import static li.strolch.agent.ComponentContainerTest.destroyContainer;
import static li.strolch.agent.ComponentContainerTest.logger;
import static li.strolch.agent.ComponentContainerTest.startContainer;
import java.util.Map;
import java.util.Map.Entry;
import li.strolch.agent.api.StrolchAgent;
import li.strolch.runtime.privilege.PrivilegeHandler;
import org.junit.Test;
import ch.eitchnet.privilege.model.Certificate;
import ch.eitchnet.utils.Version;
import ch.eitchnet.utils.collections.MapOfLists;
public class MigrationsTest {
@Test
public void shouldRunMigrations() {
try {
StrolchAgent agent = startContainer("target/MigrationsTest/", "src/test/resources/migrationstest");
PrivilegeHandler privilegeHandler = agent.getContainer().getPrivilegeHandler();
Certificate cert = privilegeHandler.authenticate("test", "test".getBytes());
MigrationsHandler migrationsHandler = agent.getContainer().getComponent(MigrationsHandler.class);
Map<String, Version> currentVersions = migrationsHandler.getCurrentVersions(cert);
for (Entry<String, Version> entry : currentVersions.entrySet()) {
logger.info("[" + entry.getKey() + "] Current version: " + entry.getValue());
}
MapOfLists<String, Version> migrationsToRun = migrationsHandler.queryMigrationsToRun(cert);
for (String realm : migrationsToRun.keySet()) {
logger.info("[" + realm + "] Migrations to run: " + migrationsToRun.getList(realm));
}
destroyContainer(agent);
} catch (Exception e) {
logger.error(e.getMessage(), e);
throw e;
}
}
}

View File

@ -1,5 +1,10 @@
package li.strolch.migrations;
import static li.strolch.migrations.Migration.BAG_PARAMETERS;
import static li.strolch.migrations.Migration.MIGRATIONS_ID;
import static li.strolch.migrations.Migration.MIGRATIONS_TYPE;
import static li.strolch.migrations.Migration.PARAM_CURRENT_VERSION;
import java.util.HashMap;
import java.util.Map;
@ -14,11 +19,6 @@ import ch.eitchnet.utils.Version;
public class CurrentMigrationVersionQuery {
public static final String MIGRATIONS_TYPE = "Migrations";
public static final String MIGRATIONS_ID = "migrations";
public static final String BAG_PARAMETERS = "parameters";
public static final String PARAM_CURRENT_VERSION = "currentVersion";
private ComponentContainer container;
private Map<String, Version> currentVersions;

View File

@ -0,0 +1,48 @@
package li.strolch.migrations;
import java.io.File;
import java.text.MessageFormat;
import java.util.Collections;
import li.strolch.agent.api.ComponentContainer;
import li.strolch.command.XmlImportModelCommand;
import li.strolch.exception.StrolchException;
import li.strolch.model.ModelStatistics;
import li.strolch.persistence.api.StrolchTransaction;
import ch.eitchnet.privilege.model.Certificate;
import ch.eitchnet.utils.Version;
public class DataMigration extends Migration {
public DataMigration(String realm, Version version, File dataFile) {
super(realm, version, dataFile);
}
@Override
public void migrate(ComponentContainer container, Certificate certificate) {
XmlImportModelCommand command;
try (StrolchTransaction tx = container.getRealm(getRealm()).openTx(certificate, DataMigration.class)) {
command = new XmlImportModelCommand(container, tx);
command.setModelFile(getDataFile());
command.setAddOrders(true);
command.setAddResources(true);
command.setUpdateOrders(true);
command.setUpdateResources(true);
command.setOrderTypes(Collections.emptySet());
command.setResourceTypes(Collections.emptySet());
tx.addCommand(command);
tx.addCommand(buildMigrationVersionChangeCommand(container, tx));
tx.commitOnClose();
} catch (Exception e) {
String msg = MessageFormat.format("Migration of {0} failed due to {1}", getVersion(), e.getMessage());
throw new StrolchException(msg, e);
}
ModelStatistics statistics = command.getStatistics();
logger.info(MessageFormat
.format("[{0}] Data migration for {1} loaded {2} Resources and {3} Orders.", getRealm(), getVersion(), statistics.nrOfResources, statistics.nrOfOrders)); //$NON-NLS-1$
}
}

View File

@ -0,0 +1,81 @@
package li.strolch.migrations;
import java.io.File;
import li.strolch.agent.api.ComponentContainer;
import li.strolch.command.AddResourceCommand;
import li.strolch.command.parameter.SetParameterCommand;
import li.strolch.model.ParameterBag;
import li.strolch.model.Resource;
import li.strolch.model.parameter.StringParameter;
import li.strolch.persistence.api.StrolchTransaction;
import li.strolch.service.api.Command;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ch.eitchnet.privilege.model.Certificate;
import ch.eitchnet.utils.Version;
public abstract class Migration {
public static final String MIGRATIONS_TYPE = "Migrations";
public static final String MIGRATIONS_ID = "migrations";
public static final String BAG_PARAMETERS = "parameters";
public static final String PARAM_CURRENT_VERSION = "currentVersion";
protected static final Logger logger = LoggerFactory.getLogger(CodeMigration.class);
protected final String realm;
protected final Version version;
protected final File dataFile;
public Migration(String realm, Version version, File dataFile) {
this.realm = realm;
this.version = version;
this.dataFile = dataFile;
}
public String getRealm() {
return realm;
}
public Version getVersion() {
return version;
}
public File getDataFile() {
return dataFile;
}
protected Command buildMigrationVersionChangeCommand(ComponentContainer container, StrolchTransaction tx) {
Resource migrationsRes = tx.getResourceBy(MIGRATIONS_TYPE, MIGRATIONS_ID);
if (migrationsRes == null) {
migrationsRes = new Resource(MIGRATIONS_ID, MIGRATIONS_TYPE, MIGRATIONS_TYPE);
ParameterBag bag = new ParameterBag(BAG_PARAMETERS, BAG_PARAMETERS, BAG_PARAMETERS);
migrationsRes.addParameterBag(bag);
StringParameter currentVersionP = new StringParameter(PARAM_CURRENT_VERSION, PARAM_CURRENT_VERSION,
getVersion().toString());
bag.addParameter(currentVersionP);
AddResourceCommand cmd = new AddResourceCommand(container, tx);
cmd.setResource(migrationsRes);
return cmd;
} else {
StringParameter currentVersionP = migrationsRes.getParameter(BAG_PARAMETERS, PARAM_CURRENT_VERSION);
SetParameterCommand cmd = new SetParameterCommand(container, tx);
cmd.setParameter(currentVersionP);
cmd.setValueAsString(getVersion().toString());
return cmd;
}
}
public abstract void migrate(ComponentContainer container, Certificate certificate);
}

View File

@ -2,7 +2,6 @@ package li.strolch.migrations;
import java.io.File;
import java.io.FileFilter;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.SortedSet;
@ -24,18 +23,27 @@ public class Migrations {
private ComponentContainer container;
private Map<String, Version> currentVersions;
private boolean verbose;
private SortedSet<DataMigration> dataMigrations;
private SortedSet<CodeMigration> codeMigrations;
private Map<String, Version> migrationsRan;
private MapOfLists<String, Version> migrationsRan;
public Migrations(ComponentContainer container, Map<String, Version> currentVersions) {
this.container = container;
this.currentVersions = currentVersions;
}
public Map<String, Version> getMigrationsRan() {
public void setVerbose(boolean verbose) {
this.verbose = verbose;
}
public boolean isVerbose() {
return verbose;
}
public MapOfLists<String, Version> getMigrationsRan() {
return this.migrationsRan;
}
@ -49,12 +57,13 @@ public class Migrations {
this.codeMigrations = loadCodeMigrations(this.currentVersions, migrationsPath);
// log found migrations
logDetectedMigrations(this.currentVersions, this.dataMigrations, this.codeMigrations);
if (this.verbose)
logDetectedMigrations(this.currentVersions, this.dataMigrations, this.codeMigrations);
}
public void runMigrations(Certificate certificate) {
Map<String, Version> migrationsRan = new HashMap<>();
MapOfLists<String, Version> migrationsRan = new MapOfLists<>();
for (Entry<String, Version> entry : this.currentVersions.entrySet()) {
String realm = entry.getKey();
@ -69,18 +78,24 @@ public class Migrations {
if (!this.codeMigrations.isEmpty()) {
for (CodeMigration migration : this.codeMigrations.tailSet(currentCodeMigration)) {
migration.migrate(container, certificate);
migrationsRan.put(realm, migration.getVersion());
migrationsRan.addElement(realm, migration.getVersion());
}
}
if (!this.dataMigrations.isEmpty()) {
for (DataMigration migration : this.dataMigrations.tailSet(currentDataMigration)) {
migration.migrate(container, certificate);
migrationsRan.put(realm, migration.getVersion());
migrationsRan.addElement(realm, migration.getVersion());
}
}
}
if (migrationsRan.isEmpty()) {
logger.info("There were no migrations required!");
} else {
logger.info("Migrated " + migrationsRan.size() + " realms!");
}
this.migrationsRan = migrationsRan;
}

View File

@ -16,8 +16,10 @@
package li.strolch.migrations;
import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
import li.strolch.agent.api.ComponentContainer;
import li.strolch.agent.api.RealmHandler;
@ -34,17 +36,30 @@ import ch.eitchnet.utils.collections.MapOfLists;
*/
public class MigrationsHandler extends StrolchComponent {
private static final String PROP_VERBOSE = "verbose";
private static final String PROP_POLL_MIGRATIONS = "pollMigrations";
private static final String PROP_POLL_WAIT = "pollWait";
private static final String PROP_RUN_MIGRATIONS_ON_START = "runMigrationsOnStart";
private static final String PATH_MIGRATIONS = "migrations";
private boolean runMigrationsOnStart;
private boolean verbose;
private Migrations migrations;
private Map<String, Version> lastMigrations;
private MapOfLists<String, Version> lastMigrations;
private File migrationsPath;
private Timer migrationTimer;
private boolean pollMigrations;
private int pollWait;
public MigrationsHandler(ComponentContainer container, String componentName) {
super(container, componentName);
}
public Map<String, Version> getLastMigrations() {
public MapOfLists<String, Version> getLastMigrations() {
if (this.lastMigrations == null)
return new MapOfLists<>();
return this.lastMigrations;
}
@ -58,25 +73,29 @@ public class MigrationsHandler extends StrolchComponent {
Map<String, Version> currentVersions = getCurrentVersions(cert);
Migrations migrations = new Migrations(getContainer(), currentVersions);
migrations.parseMigrations(this.migrationsPath);
migrations.setVerbose(this.verbose);
this.migrations = migrations;
return this.migrations.getMigrationsToRun();
}
public void runMigrations(Certificate cert) {
this.lastMigrations.clear();
queryMigrationsToRun(cert);
this.migrations.runMigrations(cert);
this.lastMigrations.putAll(this.migrations.getMigrationsRan());
this.lastMigrations = this.migrations.getMigrationsRan();
}
@Override
public void initialize(ComponentConfiguration configuration) {
this.lastMigrations = new HashMap<>();
this.runMigrationsOnStart = configuration.getBoolean(PROP_RUN_MIGRATIONS_ON_START, Boolean.FALSE);
this.verbose = configuration.getBoolean(PROP_VERBOSE, Boolean.FALSE);
this.pollMigrations = configuration.getBoolean(PROP_VERBOSE, Boolean.FALSE);
this.pollWait = configuration.getInt(PROP_VERBOSE, 5);
RuntimeConfiguration runtimeConf = configuration.getRuntimeConfiguration();
this.migrationsPath = runtimeConf.getDataDir(MigrationsHandler.class.getName(), PATH_MIGRATIONS, false);
if (this.migrationsPath.exists()) {
if (this.runMigrationsOnStart && this.migrationsPath.exists()) {
CurrentMigrationVersionQuery query = new CurrentMigrationVersionQuery(getContainer());
PrivilegeHandler privilegeHandler = getContainer().getComponent(PrivilegeHandler.class);
@ -86,25 +105,77 @@ public class MigrationsHandler extends StrolchComponent {
Migrations migrations = new Migrations(getContainer(), currentVersions);
migrations.parseMigrations(this.migrationsPath);
migrations.setVerbose(this.verbose);
this.migrations = migrations;
}
if (this.pollMigrations) {
this.migrationTimer = new Timer("MigrationTimer", true); //$NON-NLS-1$
long checkInterval = TimeUnit.MINUTES.toMillis(pollWait);
this.migrationTimer.schedule(new MigrationPollTask(), checkInterval, checkInterval);
}
super.initialize(configuration);
}
@Override
public void start() {
if (this.migrations != null) {
if (this.runMigrationsOnStart && this.migrations != null) {
PrivilegeHandler privilegeHandler = getContainer().getComponent(PrivilegeHandler.class);
RunMigrationsAction action = new RunMigrationsAction(this.migrations);
privilegeHandler.runAsSystem(RealmHandler.SYSTEM_USER_AGENT, action);
this.lastMigrations.putAll(this.migrations.getMigrationsRan());
this.lastMigrations = this.migrations.getMigrationsRan();
}
super.start();
}
@Override
public void stop() {
if (this.migrationTimer != null) {
this.migrationTimer.cancel();
}
this.migrationTimer = null;
super.stop();
}
/**
* Simpler {@link TimerTask} to check for sessions which haven't been active for
* {@link DefaultStrolchSessionHandler#PARAM_SESSION_TTL_MINUTES} minutes.
*
* @author Robert von Burg <eitch@eitchnet.ch>
*/
private class MigrationPollTask extends TimerTask {
@Override
public void run() {
CurrentMigrationVersionQuery query = new CurrentMigrationVersionQuery(getContainer());
PrivilegeHandler privilegeHandler = getContainer().getComponent(PrivilegeHandler.class);
QueryCurrentVersionsAction queryAction = new QueryCurrentVersionsAction(query);
privilegeHandler.runAsSystem(RealmHandler.SYSTEM_USER_AGENT, queryAction);
Map<String, Version> currentVersions = query.getCurrentVersions();
Migrations migrations = new Migrations(getContainer(), currentVersions);
migrations.parseMigrations(MigrationsHandler.this.migrationsPath);
migrations.setVerbose(MigrationsHandler.this.verbose);
MigrationsHandler.this.migrations = migrations;
if (migrations.getMigrationsToRun().isEmpty()) {
logger.info("There are no migrations required at the moment!");
} else {
RunMigrationsAction runMigrationsAction = new RunMigrationsAction(MigrationsHandler.this.migrations);
privilegeHandler.runAsSystem(RealmHandler.SYSTEM_USER_AGENT, runMigrationsAction);
MigrationsHandler.this.lastMigrations = MigrationsHandler.this.migrations.getMigrationsRan();
}
}
}
}

View File

@ -0,0 +1,67 @@
package li.strolch.migrations;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import li.strolch.runtime.StrolchConstants;
import li.strolch.runtime.privilege.PrivilegeHandler;
import li.strolch.testbase.runtime.RuntimeMock;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import ch.eitchnet.privilege.model.Certificate;
import ch.eitchnet.utils.Version;
import ch.eitchnet.utils.collections.MapOfLists;
public class MigrationsTest {
public static final String RUNTIME_PATH = "target/migrationstest/"; //$NON-NLS-1$
public static final String CONFIG_SRC = "src/test/resources/migrationstest"; //$NON-NLS-1$
protected static RuntimeMock runtimeMock;
protected static Certificate certificate;
@BeforeClass
public static void beforeClass() throws Exception {
File rootPath = new File(RUNTIME_PATH);
File configSrc = new File(CONFIG_SRC);
runtimeMock = new RuntimeMock();
runtimeMock.mockRuntime(rootPath, configSrc);
runtimeMock.startContainer();
certificate = runtimeMock.getPrivilegeHandler().authenticate("test", "test".getBytes());
}
@AfterClass
public static void afterClass() {
if (runtimeMock != null)
runtimeMock.destroyRuntime();
}
@Test
public void shouldRunMigrations() {
PrivilegeHandler privilegeHandler = runtimeMock.getPrivilegeHandler();
Certificate cert = privilegeHandler.authenticate("test", "test".getBytes());
MigrationsHandler migrationsHandler = runtimeMock.getContainer().getComponent(MigrationsHandler.class);
Map<String, Version> currentVersions = migrationsHandler.getCurrentVersions(cert);
assertEquals("1.1.1", currentVersions.get(StrolchConstants.DEFAULT_REALM).toString());
MapOfLists<String, Version> lastMigrations = migrationsHandler.getLastMigrations();
List<Version> expectedMigrations = Arrays.asList(Version.valueOf("0.1.0"), Version.valueOf("0.1.1"),
Version.valueOf("0.5.2"), Version.valueOf("1.0.0"), Version.valueOf("1.0.5"), Version.valueOf("1.1.1"));
assertEquals(expectedMigrations, lastMigrations.getList(StrolchConstants.DEFAULT_REALM));
MapOfLists<String, Version> migrationsToRun = migrationsHandler.queryMigrationsToRun(cert);
assertTrue("Expected to have all migrations run", migrationsToRun.isEmpty());
}
}

View File

@ -35,6 +35,8 @@ import li.strolch.testbase.runtime.RuntimeMock;
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ch.eitchnet.db.DbSchemaVersionCheck;
import ch.eitchnet.privilege.model.Certificate;
@ -51,6 +53,8 @@ public abstract class AbstractRealmServiceTest {
public static final String RUNTIME_PATH = "target/svcTestRuntime/"; //$NON-NLS-1$
public static final String CONFIG_SRC = "src/test/resources/svctest"; //$NON-NLS-1$
protected static final Logger logger = LoggerFactory.getLogger(AbstractRealmServiceTest.class);
protected static RuntimeMock runtimeMock;
protected Certificate certificate;

View File

@ -24,26 +24,23 @@
<Properties>
<realms>defaultRealm</realms>
<dataStoreMode>TRANSIENT</dataStoreMode>
<dataStoreFile>Model.xml</dataStoreFile>
<dataStoreFile>StrolchModel.xml</dataStoreFile>
</Properties>
</Component>
<Component>
<name>ServiceHandler</name>
<api>li.strolch.runtime.configuration.model.ServiceHandlerTest</api>
<impl>li.strolch.runtime.configuration.model.ServiceHandlerTestImpl</impl>
<depends>RealmHandler</depends>
<api>li.strolch.service.api.ServiceHandler</api>
<impl>li.strolch.service.api.DefaultServiceHandler</impl>
</Component>
<Component>
<name>MigrationsHandler</name>
<api>li.strolch.migrations.MigrationsHandler</api>
<impl>li.strolch.migrations.MigrationsHandler</impl>
<depends>ServiceHandler</depends>
</Component>
<Component>
<name>PostInitializer</name>
<api>li.strolch.runtime.configuration.model.PostInitializerTest</api>
<impl>li.strolch.runtime.configuration.model.PostInitializerTestImpl</impl>
<depends>MigrationsHandler</depends>
<Properties>
<verbose>true</verbose>
<runMigrationsOnStart>true</runMigrationsOnStart>
</Properties>
</Component>
</env>
</StrolchConfiguration>