Source code

001/*
002 * Copyright 2005-2007 The Kuali Foundation
003 *
004 *
005 * Licensed under the Educational Community License, Version 2.0 (the "License");
006 * you may not use this file except in compliance with the License.
007 * You may obtain a copy of the License at
008 *
009 * http://www.opensource.org/licenses/ecl2.php
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.kualigan.maven.plugins.liquibase;
018
019import java.io.PrintStream;
020import java.io.Reader;
021
022import java.lang.reflect.Field;
023
024import liquibase.database.Database;
025import liquibase.database.jvm.JdbcConnection;
026
027import java.sql.Blob;
028import java.sql.Clob;
029import java.sql.Connection;
030import java.sql.DriverManager;
031import java.sql.DatabaseMetaData;
032import java.sql.PreparedStatement;
033import java.sql.SQLException;
034import java.sql.ResultSet;
035import java.sql.ResultSetMetaData;
036import java.sql.Statement;
037import java.sql.Types;
038
039import java.util.ArrayList;
040import java.util.Collection;
041import java.util.HashMap;
042import java.util.Map;
043import java.util.Observable;
044import java.util.Observer;
045
046import org.apache.maven.plugin.MojoExecutionException;
047import org.apache.maven.plugin.logging.Log;
048
049import org.codehaus.plexus.component.annotations.Component;
050import org.codehaus.plexus.component.annotations.Requirement;
051
052
053import static org.apache.tools.ant.Project.MSG_DEBUG;
054
055/**
056 * Helper class for migrating information from one database to another
057 *
058 * @author Leo Przybylski (przybyls@arizona.edu)
059 */
060@Component(role = org.kualigan.maven.plugins.liquibase.MigrateHelper.class, hint="default")
061public class DefaultMigrateHelper implements MigrateHelper {
062    public static final String ROLE_HINT = "default";
063    
064    private static final String[] carr = new String[] {"|", "\\", "-", "/"};
065    private static final String RECORD_COUNT_QUERY = "select count(*) as \"COUNT\" from %s";
066    private static final String SELECT_ALL_QUERY   = "select * from %s";
067    private static final String INSERT_STATEMENT   = "insert into %s (%s) values (%s)";
068    private static final String DATE_CONVERSION    = "TO_DATE('%s', 'YYYYMMDDHH24MISS')";
069    private static final String COUNT_FIELD        = "COUNT";
070    private static final String LIQUIBASE_TABLE    = "DATABASECHANGELOG";
071    private static final int[]  QUOTED_TYPES       = 
072        new int[] {Types.CHAR, Types.VARCHAR, Types.TIME, Types.LONGVARCHAR, Types.DATE, Types.TIMESTAMP};
073
074    private static final String HSQLDB_PUBLIC      = "PUBLIC";
075    private static final int    MAX_THREADS        = 3;
076
077    private Log log;
078    private Database source;
079    private Database target;
080    private int threadCount;
081    private Boolean interactiveMode;
082
083    public DefaultMigrateHelper() { 
084        int threadCount = 1;
085    }
086    
087    public void setSource(final Database source) {
088        this.source = source;
089    }
090    
091    public Database getSource() {
092        return this.source;
093    }
094
095    public void setTarget(final Database target) {
096        this.target = target;
097    }
098
099    public Database getTarget() {
100        return this.target;
101    }
102    
103    public Log getLog() {
104        return log;
105    }
106    
107    public void setLog(final Log log) {
108        this.log = log;
109    }
110    
111    public void migrate(final Database source, final Database target, final Log log, final Boolean interactiveMode) throws MojoExecutionException {
112        setTarget(target);
113        setSource(source);
114        setLog(log);
115        this.interactiveMode = interactiveMode;
116        migrate();
117    }
118    
119    public void migrate() throws MojoExecutionException {
120        getLog().debug("Migrating data from " + source.getConnection().getURL() + " to " + target.getConnection().getURL());
121
122        final Incrementor recordCountIncrementor = new Incrementor();
123        final Map<String, Integer> tableData = getTableData(recordCountIncrementor);
124
125        getLog().debug("Copying " + tableData.size() + " tables");
126
127        float recordVisitor = 0;
128        final ProgressObserver progressObserver = new ProgressObserver(recordCountIncrementor.getValue(),
129                                                                       48f, 48f/100,
130                                                                       "\r|%s[%s] %3d%% (%d/%d) records");
131        final ProgressObservable observable = new ProgressObservable();
132        observable.addObserver(progressObserver);
133
134        final ThreadGroup tgroup = new ThreadGroup("Migration Threads");
135
136        for (final String tableName : tableData.keySet()) {
137        //     debug("Migrating table " + tableName + " with " + tableData.get(tableName) + " records");
138            /*
139            if (tgroup.activeCount() < MAX_THREADS) {
140                new Thread(tgroup, new Runnable() {
141                        public void run() {
142                            migrate(source, target, tableName, observable);
143                        }
144                    }).start();
145            }
146            else {
147            */
148            final Map<String,Integer> columns = new HashMap<String, Integer>();
149            migrate(tableName, observable);
150                // }
151        }
152
153        // Wait for other threads to finish
154        try {
155            while(tgroup.activeCount() > 0) {
156                Thread.sleep(5000);
157            }
158        }
159        catch (InterruptedException e) {
160        }
161
162    /*
163        try {
164            final JdbcConnection targetDb = target.getConnection();
165            if (targetDb.getMetaData().getDriverName().toLowerCase().contains("hsqldb")) {
166                Statement st = targetDb.createStatement();
167                st.execute("CHECKPOINT"); 
168                st.close();
169            }
170            targetDb.close();
171        }
172        catch (Exception e) {
173            throw new MojoExecutionException(e.getMessage(), e);
174        }
175        */
176    }
177
178    protected void migrate(final String tableName, 
179                           final ProgressObservable observable) throws MojoExecutionException {
180        final JdbcConnection sourceDb = (JdbcConnection) getSource().getConnection();
181        final JdbcConnection targetDb = (JdbcConnection) getTarget().getConnection();
182
183        final Map<String, Integer> columns = getColumnMap(tableName);
184
185        if (columns.size() < 1) {
186            getLog().debug("Columns are empty for " + tableName);
187            return;
188        }
189
190        final PreparedStatement toStatement = prepareStatement(targetDb, tableName, columns);
191        Statement fromStatement = null;
192
193        final boolean hasClob = columns.values().contains(Types.CLOB);
194        int recordsLost = 0;
195        
196        try {
197            fromStatement = sourceDb.createStatement();
198            final ResultSet results = fromStatement.executeQuery(String.format(SELECT_ALL_QUERY, tableName));
199            
200            try {
201                while (results.next()) {
202                    try {
203                        toStatement.clearParameters();
204                        
205                        int i = 1;
206                        for (String columnName : columns.keySet()) {
207                            final Object value = results.getObject(columnName);
208                            
209                            if (value != null) {
210                                try {
211                                    handleLob(toStatement, value, i);
212                                }
213                                catch (Exception e) {
214                                    if (getLog().isDebugEnabled()) {
215                                        // getLog().warn(String.format("Error processing %s.%s %s", tableName, columnName, columns.get(columnName)));
216                                        if (Clob.class.isAssignableFrom(value.getClass())) {
217                                            // getLog().warn("Got exception trying to insert CLOB with length" + ((Clob) value).length());
218                                        }
219                                        // e.printStackTrace();
220                                    }
221                                }
222                            }
223                            else {
224                                toStatement.setObject(i,value);
225                            } 
226                            i++;
227                        }
228                        
229                        boolean retry = true;
230                        int retry_count = 0;
231                        while(retry) {
232                            try {
233                                toStatement.execute();
234                                retry = false;
235                            }
236                            catch (SQLException sqle) {
237                                retry = false;
238                                if (sqle.getMessage().contains("ORA-00942")) {
239                                    getLog().debug("Couldn't find " + tableName);
240                                    if (getLog().isDebugEnabled()) {
241                                        getLog().debug("Tried insert statement " + getStatementBuffer(tableName, columns), sqle);
242                                    }
243                                }
244                                else if (sqle.getMessage().contains("ORA-12519")) {
245                                    retry = true;
246                                    if (getLog().isDebugEnabled()) {
247                                        getLog().debug("Tried insert statement " + getStatementBuffer(tableName, columns), sqle);
248                                    }
249                                }
250                                else if (sqle.getMessage().contains("IN or OUT")) {
251                                    if (getLog().isDebugEnabled()) {
252                                        getLog().debug("Column count was " + columns.keySet().size(), sqle);
253                                    }
254                                }
255                                else if (sqle.getMessage().contains("Error reading")) {
256                                    if (retry_count > 5) {
257                                        if (getLog().isDebugEnabled()) {
258                                            getLog().debug("Tried insert statement " + getStatementBuffer(tableName, columns), sqle);
259                                        }
260                                        retry = false;
261                                    }
262                                    retry_count++;
263                                }
264                                else {
265                                    if (getLog().isDebugEnabled()) {
266                                        // getLog().warn("Error executing: " + getStatementBuffer(tableName, columns), sqle);
267                                    }
268                                }
269                            }
270                        }
271                    }
272                    catch (Exception e) {
273                        recordsLost++;
274                        throw e;
275                    }
276                    finally {
277                        observable.incrementRecord();
278                    }
279                }
280            }
281            finally {
282                if (results != null) {
283                    try {
284                        results.close();
285                    }
286                    catch(Exception e) {
287                    }
288                }
289            }
290        }
291        catch (Exception e) {
292            throw new MojoExecutionException(e.getMessage(), e);
293        }
294        finally {
295            if (sourceDb != null) {
296                try {
297                    if (sourceDb.getMetaData().getDriverName().toLowerCase().contains("hsqldb")) {
298                        Statement st = sourceDb.createStatement();
299                        st.execute("CHECKPOINT"); 
300                        st.close();
301                    }
302                    fromStatement.close();
303                    //sourceDb.close();
304                }
305                catch (Exception e) {
306                }
307            }
308
309            if (targetDb != null) {
310                try {
311                    targetDb.commit();
312                    if (targetDb.getMetaData().getDriverName().toLowerCase().contains("hsql")) {
313                        Statement st = targetDb.createStatement();
314                        st.execute("CHECKPOINT"); 
315                        st.close();
316                    }
317                    toStatement.close();
318                    // targetDb.close();
319                }
320                catch (Exception e) {
321                    getLog().debug("Error closing database connection");
322                    e.printStackTrace();
323                }
324            }
325            // debug("Lost " +recordsLost + " records");
326            columns.clear();
327        }
328    }
329
330    protected void handleLob(final PreparedStatement toStatement, final Object value, final int i) throws SQLException {
331        if (Clob.class.isAssignableFrom(value.getClass())) {
332            toStatement.setAsciiStream(i, ((Clob) value).getAsciiStream(), ((Clob) value).length());
333        }
334        else if (Blob.class.isAssignableFrom(value.getClass())) {
335            toStatement.setBinaryStream(i, ((Blob) value).getBinaryStream(), ((Blob) value).length());
336        }
337        else {
338            toStatement.setObject(i,value);
339        } 
340    }
341
342    protected PreparedStatement prepareStatement(final JdbcConnection conn, 
343                                                 final String tableName, 
344                                                 final Map<String, Integer> columns) throws MojoExecutionException {
345        final String statement = getStatementBuffer(tableName, columns);
346        
347        try {
348            return conn.prepareStatement(statement);
349        }
350        catch (Exception e) {
351            throw new MojoExecutionException(e.getMessage(), e);
352        }
353    }
354
355    protected String getStatementBuffer(final String tableName, final Map<String,Integer> columns) {
356        String retval = null;
357
358        final StringBuilder names  = new StringBuilder();
359        final StringBuilder values = new StringBuilder();
360        for (String columnName : columns.keySet()) {
361            names.append(columnName).append(",");
362            values.append("?,");
363        }
364
365        names.setLength(names.length() - 1);
366        values.setLength(values.length() - 1);
367        retval = String.format(INSERT_STATEMENT, tableName, names, values);
368        
369
370        return retval;
371    }
372
373    protected boolean isValidTable(final DatabaseMetaData metadata, final String tableName) {
374        return !(tableName.startsWith("BIN$") || tableName.toUpperCase().startsWith(LIQUIBASE_TABLE) || isSequence(metadata, tableName));
375    }
376
377    protected boolean isSequence(final DatabaseMetaData metadata, final String tableName) {
378        final JdbcConnection source = (JdbcConnection) getSource().getConnection();
379        try {
380            final ResultSet rs = source.getMetaData().getColumns(null, getSource().getDefaultSchemaName(), tableName, null);
381            int columnCount = 0;
382            boolean hasId = false;
383            try {
384                while (rs.next()) {
385                    columnCount++;
386                    if ("yes".equalsIgnoreCase(rs.getString("IS_AUTOINCREMENT"))) {
387                        hasId = true;
388                    }
389                }
390            }
391            finally {
392                if (rs != null) {
393                    try {
394                        rs.close();
395                    }
396                    catch (Exception e) {
397                    }
398                }
399                return (columnCount == 1 && hasId);
400            }
401        }
402        catch (Exception e) {
403            return false;
404        }
405    }
406
407    /**
408     * Get a list of table names available mapped to row counts
409     */
410    protected Map<String, Integer> getTableData(final Incrementor incrementor) throws MojoExecutionException {
411        JdbcConnection sourceConn = (JdbcConnection) getSource().getConnection();
412        JdbcConnection targetConn = (JdbcConnection) getTarget().getConnection();
413        final Map<String, Integer> retval = new HashMap<String, Integer>();
414        final Collection<String> toRemove = new ArrayList<String>();
415
416        getLog().debug("Looking up table names in schema " + getSource().getDefaultSchemaName());
417        try {
418            final DatabaseMetaData metadata = sourceConn.getMetaData();
419            final ResultSet tableResults = metadata.getTables(sourceConn.getCatalog(), getSource().getDefaultSchemaName(), null, new String[] { "TABLE" });
420            while (tableResults.next()) {
421                final String tableName = tableResults.getString("TABLE_NAME");
422                if (!isValidTable(metadata, tableName)) {
423                    continue;
424                }
425                if (tableName.toUpperCase().startsWith(LIQUIBASE_TABLE)) continue;
426                final int rowCount = getTableRecordCount(sourceConn, tableName);
427                if (rowCount < 1) { // no point in going through tables with no data
428                    
429                }
430                incrementor.increment(rowCount);
431                // debug("Adding table " + tableName);
432                retval.put(tableName, rowCount);
433            }
434            tableResults.close();
435        }
436        catch (Exception e) {
437            throw new MojoExecutionException(e.getMessage(), e);
438        }
439
440        try {
441            for (String tableName : retval.keySet()) {
442                final ResultSet tableResults = targetConn.getMetaData().getTables(targetConn.getCatalog(), getTarget().getDefaultSchemaName(), null, new String[] { "TABLE" });
443                if (!tableResults.next()) {
444                    getLog().debug("Removing " + tableName);
445                    toRemove.add(tableName);
446                }
447                tableResults.close();
448            }
449        }
450        catch (Exception e) {
451            throw new MojoExecutionException(e.getMessage(), e);
452        }
453
454        for (String tableName : toRemove) {
455            retval.remove(tableName);
456        }
457        
458        return retval;
459    }
460
461    protected Map<String, Integer> getColumnMap(final String tableName) throws MojoExecutionException {
462        final JdbcConnection targetDb = (JdbcConnection) target.getConnection();
463        final JdbcConnection sourceDb = (JdbcConnection) source.getConnection();
464        final Map<String,Integer> retval = new HashMap<String,Integer>();
465        final Collection<String> toRemove = new ArrayList<String>();
466        try {
467            final Statement state = targetDb.createStatement();                
468            final ResultSet altResults = state.executeQuery("select * from " + tableName + " where 1 = 0");
469            final ResultSetMetaData metadata = altResults.getMetaData();
470            
471            for (int i = 1; i <= metadata.getColumnCount(); i++) {
472                retval.put(metadata.getColumnName(i),
473                           metadata.getColumnType(i));
474            }
475            altResults.close();
476            state.close();
477        }
478        catch (Exception e) {
479            throw new MojoExecutionException(e.getMessage(), e);
480        }
481
482        for (final String column : retval.keySet()) {
483            try {
484                final Statement state = targetDb.createStatement();                
485                final ResultSet altResults = state.executeQuery("select * from " + tableName + " where 1 = 0");
486                final ResultSetMetaData metadata = altResults.getMetaData();
487
488                for (int i = 1; i <= metadata.getColumnCount(); i++) {
489                    retval.put(metadata.getColumnName(i),
490                               metadata.getColumnType(i));
491                }
492                altResults.close();
493                state.close();
494            }
495            catch (Exception e) {
496                throw new MojoExecutionException(e.getMessage(), e);
497            }
498        }
499
500        for (final String column : toRemove) {
501            retval.remove(column);
502        }
503        
504        return retval;
505    }
506
507    protected int getTableRecordCount(final JdbcConnection conn, final String tableName) throws MojoExecutionException {
508        final String query = String.format(RECORD_COUNT_QUERY, tableName);
509        Statement statement = null;
510        try {
511            statement = conn.createStatement();
512            final ResultSet results = statement.executeQuery(query);
513            results.next();
514            final int retval = results.getInt(COUNT_FIELD);
515            results.close();
516            return retval;
517        }
518        catch (Exception e) {
519            if (e.getMessage().contains("ORA-00942")) {
520                getLog().debug("Couldn't find " + tableName);
521                getLog().debug("Tried insert statement " + query);
522            }
523            getLog().debug("Exception executing " + query);
524            throw new MojoExecutionException(e.getMessage(), e);
525        }
526        finally {
527            try {
528                if (statement != null) {
529                    statement.close();
530                    statement = null;
531                }
532            }
533            catch (Exception e) {
534            }
535        }
536    }
537
538    /*
539    private void debug(String msg) {
540        getLog.debug(msg, MSG_DEBUG);
541    }
542
543    private Connection openConnection(String reference) {
544        final RdbmsConfig config = (RdbmsConfig) getProject().getReference(reference);
545        return openConnection(config);
546    }
547    
548    private Connection openConnection(RdbmsConfig config) {
549        Connection retval = null;
550        
551        while (retval == null) {
552            try {
553                debug("Loading schema " + config.getSchema() + " at url " + config.getUrl());
554                Class.forName(config.getDriver());
555
556                retval = DriverManager.getConnection(config.getUrl(), config.getUsername(), config.getPassword());
557                retval.setAutoCommit(false);
558
559
560                // If this is an HSQLDB database, then we probably want to turn off logging for permformance
561                if (config.getDriver().indexOf("hsqldb") > -1) {
562                    debug("Disabling hsqldb log");
563                    final Statement st = retval.createStatement();
564                    st.execute("SET FILES LOG FALSE");
565                    st.close();
566                }
567                
568            }
569            catch (Exception e) {
570                // throw new MojoExecutionException(e.getMessage(), e);
571            }
572        }
573        
574        return retval;
575    }
576*/
577
578    /**
579     * Helper class for incrementing values
580     */
581    private class Incrementor {
582        private int value;
583        
584        public Incrementor() {
585            value = 0;
586        }
587        
588        public int getValue() {
589            return value;
590        }
591
592        public void increment() {
593            value++;
594        }
595
596        public void increment(int by) {
597            value += by;
598        }
599    }
600
601    private class ProgressObservable extends Observable {
602        public void incrementRecord() {
603            setChanged();
604            notifyObservers();
605            clearChanged();
606        }
607    }
608
609    /**
610     * Observer for handling progress
611     * 
612     */
613    private class ProgressObserver implements Observer {
614
615        private float total;
616        private float progress;
617        private float length;
618        private float ratio;
619        private String template;
620        private float count;
621        private PrintStream out;
622        
623        public ProgressObserver(final float total,
624                                final float length,
625                                final float ratio,
626                                final String template) {
627            this.total    = total;
628            this.template = template;
629            this.ratio    = ratio;
630            this.length   = length;
631            this.count    = 0;
632            
633            out = System.out;
634            /*
635            try {
636                final Field field = Main.class.getDeclaredField("out");
637                field.setAccessible(true);
638                out = (PrintStream) field.get(null);
639            }
640            catch (Exception e) {
641                e.printStackTrace();
642            }
643            */
644        }
645
646        public synchronized void update(Observable o, Object arg) {
647            count++;
648
649            final int percent = (int) ((count / total) * 100f);
650            final int progress = (int) ((count / total) * (100f * ratio));
651            final StringBuilder progressBuffer = new StringBuilder();
652                
653            for (int x = 0; x < progress; x++) {
654                progressBuffer.append('=');
655            }
656            
657            for (int x = progress; x < length; x++) {
658                progressBuffer.append(' ');
659            }
660            int roll = (int) (count / (total / 1000));
661
662            if (interactiveMode) {
663                out.print(String.format(template, progressBuffer, carr[roll % carr.length], percent, (int) count, (int) total));
664            }
665            else if ((count % 5000) == 0 || count == total) {
666                out.println(String.format("(%s)%% %s of %s records", (int) ((count / total) * 100), (int) count, (int) total));
667            }            
668        }
669    }
670}