DataX的DBUtil依赖的一些类如下图：
RetryUtil 失败重连，测试过只要把连接地址url输入错误，即可生效
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.concurrent.*;
public final class RetryUtil {
private static final Logger LOG = LoggerFactory.getLogger(RetryUtil.class);
private static final long MAX_SLEEP_MILLISECOND = 256 * 1000;
/**
* 重试次数工具方法.
*
* @param callable 实际逻辑
* @param retryTimes 最大重试次数（>1）
* @param sleepTimeInMilliSecond 运行失败后休眠对应时间再重试
* @param exponential 休眠时间是否指数递增
* @param <T> 返回值类型
* @return 经过重试的callable的执行结果
*/
public static <T> T executeWithRetry(Callable<T> callable,
int retryTimes,
long sleepTimeInMilliSecond,
boolean exponential) throws Exception {
Retry retry = new Retry();
return retry.doRetry(callable, retryTimes, sleepTimeInMilliSecond, exponential, null);
}
/**
* 重试次数工具方法.
*
* @param callable 实际逻辑
* @param retryTimes 最大重试次数（>1）
* @param sleepTimeInMilliSecond 运行失败后休眠对应时间再重试
* @param exponential 休眠时间是否指数递增
* @param <T> 返回值类型
* @param retryExceptionClasss 出现指定的异常类型时才进行重试
* @return 经过重试的callable的执行结果
*/
public static <T> T executeWithRetry(Callable<T> callable,
int retryTimes,
long sleepTimeInMilliSecond,
boolean exponential,
List<Class<?>> retryExceptionClasss) throws Exception {
Retry retry = new Retry();
return retry.doRetry(callable, retryTimes, sleepTimeInMilliSecond, exponential, retryExceptionClasss);
}
/**
* 在外部线程执行并且重试。每次执行需要在timeoutMs内执行完，不然视为失败。
* 执行异步操作的线程池从外部传入，线程池的共享粒度由外部控制。比如，HttpClientUtil共享一个线程池。
* <p/>
* 限制条件：仅仅能够在阻塞的时候interrupt线程
*
* @param callable 实际逻辑
* @param retryTimes 最大重试次数（>1）
* @param sleepTimeInMilliSecond 运行失败后休眠对应时间再重试
* @param exponential 休眠时间是否指数递增
* @param timeoutMs callable执行超时时间，毫秒
* @param executor 执行异步操作的线程池
* @param <T> 返回值类型
* @return 经过重试的callable的执行结果
*/
public static <T> T asyncExecuteWithRetry(Callable<T> callable,
int retryTimes,
long sleepTimeInMilliSecond,
boolean exponential,
long timeoutMs,
ThreadPoolExecutor executor) throws Exception {
Retry retry = new AsyncRetry(timeoutMs, executor);
return retry.doRetry(callable, retryTimes, sleepTimeInMilliSecond, exponential, null);
}
/**
* 创建异步执行的线程池。特性如下：
* core大小为0，初始状态下无线程，无初始消耗。
* max大小为5，最多五个线程。
* 60秒超时时间，闲置超过60秒线程会被回收。
* 使用SynchronousQueue，任务不会排队，必须要有可用线程才能提交成功，否则会RejectedExecutionException。
*
* @return 线程池
*/
public static ThreadPoolExecutor createThreadPoolExecutor() {
return new ThreadPoolExecutor(0, 5,
60L, TimeUnit.SECONDS,
new SynchronousQueue<Runnable>());
}
private static class Retry {
public <T> T doRetry(Callable<T> callable, int retryTimes, long sleepTimeInMilliSecond, boolean exponential, List<Class<?>> retryExceptionClasss)
throws Exception {
if (null == callable) {
throw new IllegalArgumentException("系统编程错误, 入参callable不能为空 ! ");
}
if (retryTimes < 1) {
throw new IllegalArgumentException(String.format(
"系统编程错误, 入参retrytime[%d]不能小于1 !", retryTimes));
}
Exception saveException = null;
for (int i = 0; i < retryTimes; i++) {
try {
return call(callable);
} catch (Exception e) {
saveException = e;
if (i == 0) {
LOG.error(String.format("Exception when calling callable, 异常Msg:%s", saveException.getMessage()), saveException);
}
if (null != retryExceptionClasss && !retryExceptionClasss.isEmpty()) {
boolean needRetry = false;
for (Class<?> eachExceptionClass : retryExceptionClasss) {
if (eachExceptionClass == e.getClass()) {
needRetry = true;
break;
}
}
if (!needRetry) {
throw saveException;
}
}
if (i + 1 < retryTimes && sleepTimeInMilliSecond > 0) {
long startTime = System.currentTimeMillis();
long timeToSleep;
if (exponential) {
timeToSleep = sleepTimeInMilliSecond * (long) Math.pow(2, i);
if(timeToSleep >= MAX_SLEEP_MILLISECOND) {
timeToSleep = MAX_SLEEP_MILLISECOND;
}
} else {
timeToSleep = sleepTimeInMilliSecond;
if(timeToSleep >= MAX_SLEEP_MILLISECOND) {
timeToSleep = MAX_SLEEP_MILLISECOND;
}
}
try {
Thread.sleep(timeToSleep);
} catch (InterruptedException ignored) {
}
long realTimeSleep = System.currentTimeMillis()-startTime;
LOG.error(String.format("Exception when calling callable, 即将尝试执行第%s次重试.本次重试计划等待[%s]ms,实际等待[%s]ms, 异常Msg:[%s]",
i+1, timeToSleep,realTimeSleep, e.getMessage()));
}
}
}
throw saveException;
}
protected <T> T call(Callable<T> callable) throws Exception {
return callable.call();
}
}
private static class AsyncRetry extends Retry {
private long timeoutMs;
private ThreadPoolExecutor executor;
public AsyncRetry(long timeoutMs, ThreadPoolExecutor executor) {
this.timeoutMs = timeoutMs;
this.executor = executor;
}
/**
* 使用传入的线程池异步执行任务，并且等待。
* <p/>
* future.get()方法，等待指定的毫秒数。如果任务在超时时间内结束，则正常返回。
* 如果抛异常（可能是执行超时、执行异常、被其他线程cancel或interrupt），都记录日志并且网上抛异常。
* 正常和非正常的情况都会判断任务是否结束，如果没有结束，则cancel任务。cancel参数为true，表示即使
* 任务正在执行，也会interrupt线程。
*
* @param callable
* @param <T>
* @return
* @throws Exception
*/
@Override
protected <T> T call(Callable<T> callable) throws Exception {
Future<T> future = executor.submit(callable);
try {
return future.get(timeoutMs, TimeUnit.MILLISECONDS);
} catch (Exception e) {
LOG.warn("Try once failed", e);
throw e;
} finally {
if (!future.isDone()) {
future.cancel(true);
LOG.warn("Try once task not done, cancel it, active count: " + executor.getActiveCount());
}
}
}
}
}
DBUtil
package cn.com.bdscheduler.integration.util.db;
import com.alibaba.druid.sql.parser.SQLParserUtils;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutableTriple;
import org.apache.commons.lang3.tuple.Triple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.sql.*;
import java.util.*;
import java.util.concurrent.*;
public final class DBUtil {
private static final Logger LOG = LoggerFactory.getLogger(DBUtil.class);
private static final ThreadLocal<ExecutorService> rsExecutors = new ThreadLocal<ExecutorService>() {
@Override
protected ExecutorService initialValue() {
return Executors.newFixedThreadPool(1, new ThreadFactoryBuilder()
.setNameFormat("rsExecutors-%d")
.setDaemon(true)
.build());
}
};
private DBUtil() {
}
public static String chooseJdbcUrl(final DataBaseType dataBaseType,
final List<String> jdbcUrls, final String username,
final String password, final List<String> preSql,
final boolean checkSlave) {
if (null == jdbcUrls || jdbcUrls.isEmpty()) {
throw DataXException.asDataXException(
DBUtilErrorCode.CONF_ERROR,
String.format("您的jdbcUrl的配置信息有错, 因为jdbcUrl[%s]不能为空. 请检查您的配置并作出修改.",
StringUtils.join(jdbcUrls, ",")));
}
try {
return RetryUtil.executeWithRetry(new Callable<String>() {
@Override
public String call() throws Exception {
boolean connOK = false;
for (String url : jdbcUrls) {
if (StringUtils.isNotBlank(url)) {
url = url.trim();
if (null != preSql && !preSql.isEmpty()) {
connOK = testConnWithoutRetry(dataBaseType,
url, username, password, preSql);
} else {
connOK = testConnWithoutRetry(dataBaseType,
url, username, password, checkSlave);
}
if (connOK) {
return url;
}
}
}
throw new Exception("DataX无法连接对应的数据库，可能原因是：1) 配置的ip/port/database/jdbc错误，无法连接。2) 配置的username/password错误，鉴权失败。请和DBA确认该数据库的连接信息是否正确。");
// throw new Exception(DBUtilErrorCode.JDBC_NULL.toString());
}
}, 7, 1000L, true);
//warn: 7 means 2 minutes
} catch (Exception e) {
throw DataXException.asDataXException(
DBUtilErrorCode.CONN_DB_ERROR,
String.format("数据库连接失败. 因为根据您配置的连接信息,无法从:%s 中找到可连接的jdbcUrl. 请检查您的配置并作出修改.",
StringUtils.join(jdbcUrls, ",")), e);
}
}
public static String chooseJdbcUrlWithoutRetry(final DataBaseType dataBaseType,
final List<String> jdbcUrls, final String username,
final String password, final List<String> preSql,
final boolean checkSlave) throws DataXException {
if (null == jdbcUrls || jdbcUrls.isEmpty()) {
throw DataXException.asDataXException(
DBUtilErrorCode.CONF_ERROR,
String.format("您的jdbcUrl的配置信息有错, 因为jdbcUrl[%s]不能为空. 请检查您的配置并作出修改.",
StringUtils.join(jdbcUrls, ",")));
}
boolean connOK = false;
for (String url : jdbcUrls) {
if (StringUtils.isNotBlank(url)) {
url = url.trim();
if (null != preSql && !preSql.isEmpty()) {
connOK = testConnWithoutRetry(dataBaseType,
url, username, password, preSql);
} else {
try {
connOK = testConnWithoutRetry(dataBaseType,
url, username, password, checkSlave);
} catch (Exception e) {
throw DataXException.asDataXException(
DBUtilErrorCode.CONN_DB_ERROR,
String.format("数据库连接失败. 因为根据您配置的连接信息,无法从:%s 中找到可连接的jdbcUrl. 请检查您的配置并作出修改.",
StringUtils.join(jdbcUrls, ",")), e);
}
}
if (connOK) {
return url;
}
}
}
throw DataXException.asDataXException(
DBUtilErrorCode.CONN_DB_ERROR,
String.format("数据库连接失败. 因为根据您配置的连接信息,无法从:%s 中找到可连接的jdbcUrl. 请检查您的配置并作出修改.",
StringUtils.join(jdbcUrls, ",")));
}
/**
* 检查slave的库中的数据是否已到凌晨00:00
* 如果slave同步的数据还未到00:00返回false
* 否则范围true
*
* @author ZiChi
* @version 1.0 2014-12-01
*/
private static boolean isSlaveBehind(Connection conn) {
try {
ResultSet rs = query(conn, "SHOW VARIABLES LIKE 'read_only'");
if (DBUtil.asyncResultSetNext(rs)) {
String readOnly = rs.getString("Value");
if ("ON".equalsIgnoreCase(readOnly)) { //备库
ResultSet rs1 = query(conn, "SHOW SLAVE STATUS");
if (DBUtil.asyncResultSetNext(rs1)) {
String ioRunning = rs1.getString("Slave_IO_Running");
String sqlRunning = rs1.getString("Slave_SQL_Running");
long secondsBehindMaster = rs1.getLong("Seconds_Behind_Master");
if ("Yes".equalsIgnoreCase(ioRunning) && "Yes".equalsIgnoreCase(sqlRunning)) {
ResultSet rs2 = query(conn, "SELECT TIMESTAMPDIFF(SECOND, CURDATE(), NOW())");
DBUtil.asyncResultSetNext(rs2);
long secondsOfDay = rs2.getLong(1);
return secondsBehindMaster > secondsOfDay;
} else {
return true;
}
} else {
LOG.warn("SHOW SLAVE STATUS has no result");
}
}
} else {
LOG.warn("SHOW VARIABLES like 'read_only' has no result");
}
} catch (Exception e) {
LOG.warn("checkSlave failed, errorMessage:[{}].", e.getMessage());
}
return false;
}
/**
* 检查表是否具有insert 权限
* insert on *.* 或者 insert on database.* 时验证通过
* 当insert on database.tableName时，确保tableList中的所有table有insert 权限，验证通过
* 其它验证都不通过
*
* @author ZiChi
* @version 1.0 2015-01-28
*/
public static boolean hasInsertPrivilege(DataBaseType dataBaseType, String jdbcURL, String userName, String password, List<String> tableList) {
/*准备参数*/
String[] urls = jdbcURL.split("/");
String dbName;
if (urls != null && urls.length != 0) {
dbName = urls[3];
}else{
return false;
}
String dbPattern = "`" + dbName + "`.*";
Collection<String> tableNames = new HashSet<String>(tableList.size());
tableNames.addAll(tableList);
Connection connection = connect(dataBaseType, jdbcURL, userName, password);
try {
ResultSet rs = query(connection, "SHOW GRANTS FOR " + userName);
while (DBUtil.asyncResultSetNext(rs)) {
String grantRecord = rs.getString("Grants for " + userName + "@%");
String[] params = grantRecord.split("\\`");
if (params != null && params.length >= 3) {
String tableName = params[3];
if (params[0].contains("INSERT") && !tableName.equals("*") && tableNames.contains(tableName))
tableNames.remove(tableName);
} else {
if (grantRecord.contains("INSERT") ||grantRecord.contains("ALL PRIVILEGES")) {
if (grantRecord.contains("*.*"))
return true;
else if (grantRecord.contains(dbPattern)) {
return true;
}
}
}
}
} catch (Exception e) {
LOG.warn("Check the database has the Insert Privilege failed, errorMessage:[{}]", e.getMessage());
}
if (tableNames.isEmpty())
return true;
return false;
}
public static boolean checkInsertPrivilege(DataBaseType dataBaseType, String jdbcURL, String userName, String password, List<String> tableList) {
Connection connection = connect(dataBaseType, jdbcURL, userName, password);
String insertTemplate = "insert into %s(select * from %s where 1 = 2)";
boolean hasInsertPrivilege = true;
Statement insertStmt = null;
for(String tableName : tableList) {
String checkInsertPrivilegeSql = String.format(insertTemplate, tableName, tableName);
try {
insertStmt = connection.createStatement();
executeSqlWithoutResultSet(insertStmt, checkInsertPrivilegeSql);
} catch (Exception e) {
if(DataBaseType.Oracle.equals(dataBaseType)) {
if(e.getMessage() != null && e.getMessage().contains("insufficient privileges")) {
hasInsertPrivilege = false;
LOG.warn("User [" + userName +"] has no 'insert' privilege on table[" + tableName + "], errorMessage:[{}]", e.getMessage());
}
} else {
hasInsertPrivilege = false;
LOG.warn("User [" + userName + "] has no 'insert' privilege on table[" + tableName + "], errorMessage:[{}]", e.getMessage());
}
}
}
try {
connection.close();
} catch (SQLException e) {
LOG.warn("connection close failed, " + e.getMessage());
}
return hasInsertPrivilege;
}
public static boolean checkDeletePrivilege(DataBaseType dataBaseType,String jdbcURL, String userName, String password, List<String> tableList) {
Connection connection = connect(dataBaseType, jdbcURL, userName, password);
String deleteTemplate = "delete from %s WHERE 1 = 2";
boolean hasInsertPrivilege = true;
Statement deleteStmt = null;
for(String tableName : tableList) {
String checkDeletePrivilegeSQL = String.format(deleteTemplate, tableName);
try {
deleteStmt = connection.createStatement();
executeSqlWithoutResultSet(deleteStmt, checkDeletePrivilegeSQL);
} catch (Exception e) {
hasInsertPrivilege = false;
LOG.warn("User [" + userName +"] has no 'delete' privilege on table[" + tableName + "], errorMessage:[{}]", e.getMessage());
}
}
try {
connection.close();
} catch (SQLException e) {
LOG.warn("connection close failed, " + e.getMessage());
}
return hasInsertPrivilege;
}
public static boolean needCheckDeletePrivilege(Configuration originalConfig) {
List<String> allSqls =new ArrayList<String>();
List<String> preSQLs = originalConfig.getList(Key.PRE_SQL, String.class);
List<String> postSQLs = originalConfig.getList(Key.POST_SQL, String.class);
if (preSQLs != null && !preSQLs.isEmpty()){
allSqls.addAll(preSQLs);
}
if (postSQLs != null && !postSQLs.isEmpty()){
allSqls.addAll(postSQLs);
}
for(String sql : allSqls) {
if(StringUtils.isNotBlank(sql)) {
if (sql.trim().toUpperCase().startsWith("DELETE")) {
return true;
}
}
}
return false;
}
/**
* Get direct JDBC connection
* <p/>
* if connecting failed, try to connect for MAX_TRY_TIMES times
* <p/>
* NOTE: In DataX, we don't need connection pool in fact
*/
public static Connection getConnection(final DataBaseType dataBaseType,
final String jdbcUrl, final String username, final String password) {
return getConnection(dataBaseType, jdbcUrl, username, password, String.valueOf(Constant.SOCKET_TIMEOUT_INSECOND * 1000));
}
/**
*
* @param dataBaseType
* @param jdbcUrl
* @param username
* @param password
* @param socketTimeout 设置socketTimeout，单位ms，String类型
* @return
*/
public static Connection getConnection(final DataBaseType dataBaseType,
final String jdbcUrl, final String username, final String password, final String socketTimeout) {
try {
return RetryUtil.executeWithRetry(new Callable<Connection>() {
@Override
public Connection call() throws Exception {
return DBUtil.connect(dataBaseType, jdbcUrl, username,
password, socketTimeout);
}
}, 9, 1000L, true);
} catch (Exception e) {
throw DataXException.asDataXException(
DBUtilErrorCode.CONN_DB_ERROR,
String.format("数据库连接失败. 因为根据您配置的连接信息:%s获取数据库连接失败. 请检查您的配置并作出修改.", jdbcUrl), e);
}
}
/**
* Get direct JDBC connection
* <p/>
* if connecting failed, try to connect for MAX_TRY_TIMES times
* <p/>
* NOTE: In DataX, we don't need connection pool in fact
*/
public static Connection getConnectionWithoutRetry(final DataBaseType dataBaseType,
final String jdbcUrl, final String username, final String password) {
return getConnectionWithoutRetry(dataBaseType, jdbcUrl, username,
password, String.valueOf(Constant.SOCKET_TIMEOUT_INSECOND * 1000));
}
public static Connection getConnectionWithoutRetry(final DataBaseType dataBaseType,
final String jdbcUrl, final String username, final String password, String socketTimeout) {
return DBUtil.connect(dataBaseType, jdbcUrl, username,
password, socketTimeout);
}
private static synchronized Connection connect(DataBaseType dataBaseType,
String url, String user, String pass) {
return connect(dataBaseType, url, user, pass, String.valueOf(Constant.SOCKET_TIMEOUT_INSECOND * 1000));
}
private static synchronized Connection connect(DataBaseType dataBaseType,
String url, String user, String pass, String socketTimeout) {
//ob10的处理
// if (url.startsWith(com.alibaba.datax.plugin.rdbms.writer.Constant.OB10_SPLIT_STRING)) {
// String[] ss = url.split(com.alibaba.datax.plugin.rdbms.writer.Constant.OB10_SPLIT_STRING_PATTERN);
// if (ss.length != 3) {
// throw DataXException
// .asDataXException(
// DBUtilErrorCode.JDBC_OB10_ADDRESS_ERROR, "JDBC OB10格式错误，请联系askdatax");
// }
// LOG.info("this is ob1_0 jdbc url.");
// user = ss[1].trim() +":"+user;
// url = ss[2].replace("jdbc:mysql:", "jdbc:oceanbase:");
// LOG.info("this is ob1_0 jdbc url. user="+user+" :url="+url);
// }
Properties prop = new Properties();
prop.put("user", user);
prop.put("password", pass);
if (dataBaseType == DataBaseType.Oracle) {
//oracle.net.READ_TIMEOUT for jdbc versions < 10.1.0.5 oracle.jdbc.ReadTimeout for jdbc versions >=10.1.0.5
// unit ms
prop.put("oracle.jdbc.ReadTimeout", socketTimeout);
}
return connect(dataBaseType, url, prop);
}
private static synchronized Connection connect(DataBaseType dataBaseType,
String url, Properties prop) {
try {
Class.forName(dataBaseType.getDriverClassName());
DriverManager.setLoginTimeout(Constant.TIMEOUT_SECONDS);
return DriverManager.getConnection(url, prop);
} catch (Exception e) {
throw RdbmsException.asConnException(dataBaseType, e, prop.getProperty("user"), null);
}
}
/**
* a wrapped method to execute select-like sql statement .
*
* @param conn Database connection .
* @param sql sql statement to be executed
* @return a {@link ResultSet}
* @throws SQLException if occurs SQLException.
*/
public static ResultSet query(Connection conn, String sql, int fetchSize)
throws SQLException {
// 默认3600 s 的query Timeout
return query(conn, sql, fetchSize, Constant.SOCKET_TIMEOUT_INSECOND);
}
/**
* a wrapped method to execute select-like sql statement .
*
* @param conn Database connection .
* @param sql sql statement to be executed
* @param fetchSize
* @param queryTimeout unit:second
* @return
* @throws SQLException
*/
public static ResultSet query(Connection conn, String sql, int fetchSize, int queryTimeout)
throws SQLException {
// make sure autocommit is off
conn.setAutoCommit(false);
Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY);
stmt.setFetchSize(fetchSize);
stmt.setQueryTimeout(queryTimeout);
return query(stmt, sql);
}
/**
* a wrapped method to execute select-like sql statement .
*
* @param stmt {@link Statement}
* @param sql sql statement to be executed
* @return a {@link ResultSet}
* @throws SQLException if occurs SQLException.
*/
public static ResultSet query(Statement stmt, String sql)
throws SQLException {
return stmt.executeQuery(sql);
}
public static void executeSqlWithoutResultSet(Statement stmt, String sql)
throws SQLException {
stmt.execute(sql);
}
/**
* Close {@link ResultSet}, {@link Statement} referenced by this
* {@link ResultSet}
*
* @param rs {@link ResultSet} to be closed
* @throws IllegalArgumentException
*/
public static void closeResultSet(ResultSet rs) {
try {
if (null != rs) {
Statement stmt = rs.getStatement();
if (null != stmt) {
stmt.close();
stmt = null;
}
rs.close();
}
rs = null;
} catch (SQLException e) {
throw new IllegalStateException(e);
}
}
public static void closeDBResources(ResultSet rs, Statement stmt,
Connection conn) {
if (null != rs) {
try {
rs.close();
} catch (SQLException unused) {
}
}
if (null != stmt) {
try {
stmt.close();
} catch (SQLException unused) {
}
}
if (null != conn) {
try {
conn.close();
} catch (SQLException unused) {
}
}
}
public static void closeDBResources(Statement stmt, Connection conn) {
closeDBResources(null, stmt, conn);
}
public static List<String> getTableColumns(DataBaseType dataBaseType,
String jdbcUrl, String user, String pass, String tableName) {
Connection conn = getConnection(dataBaseType, jdbcUrl, user, pass);
return getTableColumnsByConn(dataBaseType, conn, tableName, "jdbcUrl:"+jdbcUrl);
}
public static List<String> getTableColumnsByConn(DataBaseType dataBaseType, Connection conn, String tableName, String basicMsg) {
List<String> columns = new ArrayList<String>();
Statement statement = null;
ResultSet rs = null;
String queryColumnSql = null;
try {
statement = conn.createStatement();
queryColumnSql = String.format("select * from %s where 1=2",
tableName);
rs = statement.executeQuery(queryColumnSql);
ResultSetMetaData rsMetaData = rs.getMetaData();
for (int i = 0, len = rsMetaData.getColumnCount(); i < len; i++) {
columns.add(rsMetaData.getColumnName(i + 1));
}
} catch (SQLException e) {
throw RdbmsException.asQueryException(dataBaseType,e,queryColumnSql,tableName,null);
} finally {
DBUtil.closeDBResources(rs, statement, conn);
}
return columns;
}
/**
* @return Left:ColumnName Middle:ColumnType Right:ColumnTypeName
*/
public static Triple<List<String>, List<Integer>, List<String>> getColumnMetaData(
DataBaseType dataBaseType, String jdbcUrl, String user,
String pass, String tableName, String column) {
Connection conn = null;
try {
conn = getConnection(dataBaseType, jdbcUrl, user, pass);
return getColumnMetaData(conn, tableName, column);
} finally {
DBUtil.closeDBResources(null, null, conn);
}
}
/**
* @return Left:ColumnName Middle:ColumnType Right:ColumnTypeName
*/
public static Triple<List<String>, List<Integer>, List<String>> getColumnMetaData(
Connection conn, String tableName, String column) {
Statement statement = null;
ResultSet rs = null;
Triple<List<String>, List<Integer>, List<String>> columnMetaData = new ImmutableTriple<List<String>, List<Integer>, List<String>>(
new ArrayList<String>(), new ArrayList<Integer>(),
new ArrayList<String>());
try {
statement = conn.createStatement();
String queryColumnSql = "select " + column + " from " + tableName
+ " where 1=2";
rs = statement.executeQuery(queryColumnSql);
ResultSetMetaData rsMetaData = rs.getMetaData();
for (int i = 0, len = rsMetaData.getColumnCount(); i < len; i++) {
columnMetaData.getLeft().add(rsMetaData.getColumnName(i + 1));
columnMetaData.getMiddle().add(rsMetaData.getColumnType(i + 1));
columnMetaData.getRight().add(
rsMetaData.getColumnTypeName(i + 1));
}
return columnMetaData;
} catch (SQLException e) {
throw DataXException
.asDataXException(DBUtilErrorCode.GET_COLUMN_INFO_FAILED,
String.format("获取表:%s 的字段的元信息时失败. 请联系 DBA 核查该库、表信息.", tableName), e);
} finally {
DBUtil.closeDBResources(rs, statement, null);
}
}
public static boolean testConnWithoutRetry(DataBaseType dataBaseType,
String url, String user, String pass, boolean checkSlave){
Connection connection = null;
try {
connection = connect(dataBaseType, url, user, pass);
if (connection != null) {
if (dataBaseType.equals(dataBaseType.MySql) && checkSlave) {
//dataBaseType.MySql
boolean connOk = !isSlaveBehind(connection);
return connOk;
} else {
return true;
}
}
} catch (Exception e) {
LOG.warn("test connection of [{}] failed, for {}.", url,
e.getMessage());
} finally {
DBUtil.closeDBResources(null, connection);
}
return false;
}
public static boolean testConnWithoutRetry(DataBaseType dataBaseType,
String url, String user, String pass, List<String> preSql) {
Connection connection = null;
try {
connection = connect(dataBaseType, url, user, pass);
if (null != connection) {
for (String pre : preSql) {
if (doPreCheck(connection, pre) == false) {
LOG.warn("doPreCheck failed.");
return false;
}
}
return true;
}
} catch (Exception e) {
LOG.warn("test connection of [{}] failed, for {}.", url,
e.getMessage());
} finally {
DBUtil.closeDBResources(null, connection);
}
return false;
}
public static boolean isOracleMaster(final String url, final String user, final String pass) {
try {
return RetryUtil.executeWithRetry(new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
Connection conn = null;
try {
conn = connect(DataBaseType.Oracle, url, user, pass);
ResultSet rs = query(conn, "select DATABASE_ROLE from V$DATABASE");
if (DBUtil.asyncResultSetNext(rs, 5)) {
String role = rs.getString("DATABASE_ROLE");
return "PRIMARY".equalsIgnoreCase(role);
}
throw DataXException.asDataXException(DBUtilErrorCode.RS_ASYNC_ERROR,
String.format("select DATABASE_ROLE from V$DATABASE failed,请检查您的jdbcUrl:%s.", url));
} finally {
DBUtil.closeDBResources(null, conn);
}
}
}, 3, 1000L, true);
} catch (Exception e) {
throw DataXException.asDataXException(DBUtilErrorCode.CONN_DB_ERROR,
String.format("select DATABASE_ROLE from V$DATABASE failed, url: %s", url), e);
}
}
public static ResultSet query(Connection conn, String sql)
throws SQLException {
Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY,
ResultSet.CONCUR_READ_ONLY);
//默认3600 seconds
stmt.setQueryTimeout(Constant.SOCKET_TIMEOUT_INSECOND);
return query(stmt, sql);
}
public static List<Map<String, Object>> convertList(ResultSet rs) {
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
try {
ResultSetMetaData md = rs.getMetaData();
int columnCount = md.getColumnCount();
while (rs.next()) {
Map<String, Object> rowData = new HashMap<String, Object>();
for (int i = 1; i <= columnCount; i++) {
rowData.put(md.getColumnName(i), rs.getObject(i));
}
list.add(rowData);
}
} catch (SQLException e) {
throw new IllegalStateException(e);
} finally {
closeResultSet(rs);
}
return list;
}
private static boolean doPreCheck(Connection conn, String pre) {
ResultSet rs = null;
try {
rs = query(conn, pre);
int checkResult = -1;
if (DBUtil.asyncResultSetNext(rs)) {
checkResult = rs.getInt(1);
if (DBUtil.asyncResultSetNext(rs)) {
LOG.warn(
"pre check failed. It should return one result:0, pre:[{}].",
pre);
return false;
}
}
if (0 == checkResult) {
return true;
}
LOG.warn(
"pre check failed. It should return one result:0, pre:[{}].",
pre);
} catch (Exception e) {
LOG.warn("pre check failed. pre:[{}], errorMessage:[{}].", pre,
e.getMessage());
} finally {
DBUtil.closeResultSet(rs);
}
return false;
}
// warn:until now, only oracle need to handle session config.
public static void dealWithSessionConfig(Connection conn,
Configuration config, DataBaseType databaseType, String message) {
List<String> sessionConfig = null;
switch (databaseType) {
case Oracle:
sessionConfig = config.getList(Key.SESSION,
new ArrayList<String>(), String.class);
DBUtil.doDealWithSessionConfig(conn, sessionConfig, message);
break;
case DRDS:
// 用于关闭 drds 的分布式事务开关
sessionConfig = new ArrayList<String>();
sessionConfig.add("set transaction policy 4");
doDealWithSessionConfig(conn, sessionConfig, message);
break;
case MySql:
sessionConfig = config.getList(Key.SESSION,
new ArrayList<String>(), String.class);
DBUtil.doDealWithSessionConfig(conn, sessionConfig, message);
break;
default:
break;
}
}
private static void doDealWithSessionConfig(Connection conn,
List<String> sessions, String message) {
if (null == sessions || sessions.isEmpty()) {
return;
}
Statement stmt;
try {
stmt = conn.createStatement();
} catch (SQLException e) {
throw DataXException
.asDataXException(DBUtilErrorCode.SET_SESSION_ERROR, String
.format("session配置有误. 因为根据您的配置执行 session 设置失败. 上下文信息是:[%s]. 请检查您的配置并作出修改.", message),
e);
}
for (String sessionSql : sessions) {
LOG.info("execute sql:[{}]", sessionSql);
try {
DBUtil.executeSqlWithoutResultSet(stmt, sessionSql);
} catch (SQLException e) {
throw DataXException.asDataXException(
DBUtilErrorCode.SET_SESSION_ERROR, String.format(
"session配置有误. 因为根据您的配置执行 session 设置失败. 上下文信息是:[%s]. 请检查您的配置并作出修改.", message), e);
}
}
DBUtil.closeDBResources(stmt, null);
}
public static void sqlValid(String sql, DataBaseType dataBaseType){
SQLStatementParser statementParser = SQLParserUtils.createSQLStatementParser(sql,dataBaseType.getTypeName());
statementParser.parseStatementList();
}
/**
* 异步获取resultSet的next(),注意，千万不能应用在数据的读取中。只能用在meta的获取
* @param resultSet
* @return
*/
public static boolean asyncResultSetNext(final ResultSet resultSet) {
return asyncResultSetNext(resultSet, 3600);
}
public static boolean asyncResultSetNext(final ResultSet resultSet, int timeout) {
Future<Boolean> future = rsExecutors.get().submit(new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return resultSet.next();
}
});
try {
return future.get(timeout, TimeUnit.SECONDS);
} catch (Exception e) {
throw DataXException.asDataXException(
DBUtilErrorCode.RS_ASYNC_ERROR, "异步获取ResultSet失败", e);
}
}
public static void loadDriverClass(String pluginType, String pluginName) {
try {
String pluginJsonPath = StringUtils.join(
new String[] { System.getProperty("datax.home"), "plugin",
pluginType,
String.format("%s%s", pluginName, pluginType),
"plugin.json" }, File.separator);
Configuration configuration = Configuration.from(new File(
pluginJsonPath));
List<String> drivers = configuration.getList("drivers",
String.class);
for (String driver : drivers) {
Class.forName(driver);
}
} catch (ClassNotFoundException e) {
throw DataXException.asDataXException(DBUtilErrorCode.CONF_ERROR,
"数据库驱动加载错误, 请确认libs目录有驱动jar包且plugin.json中drivers配置驱动类正确!",
e);
}
}
}
DBUtilErrorCode
//TODO
public enum DBUtilErrorCode implements ErrorCode {
//连接错误
MYSQL_CONN_USERPWD_ERROR("MYSQLErrCode-01","数据库用户名或者密码错误，请检查填写的账号密码或者联系DBA确认账号和密码是否正确"),
MYSQL_CONN_IPPORT_ERROR("MYSQLErrCode-02","数据库服务的IP地址或者Port错误，请检查填写的IP地址和Port或者联系DBA确认IP地址和Port是否正确。如果是同步中心用户请联系DBA确认idb上录入的IP和PORT信息和数据库的当前实际信息是一致的"),
MYSQL_CONN_DB_ERROR("MYSQLErrCode-03","数据库名称错误，请检查数据库实例名称或者联系DBA确认该实例是否存在并且在正常服务"),
ORACLE_CONN_USERPWD_ERROR("ORACLEErrCode-01","数据库用户名或者密码错误，请检查填写的账号密码或者联系DBA确认账号和密码是否正确"),
ORACLE_CONN_IPPORT_ERROR("ORACLEErrCode-02","数据库服务的IP地址或者Port错误，请检查填写的IP地址和Port或者联系DBA确认IP地址和Port是否正确。如果是同步中心用户请联系DBA确认idb上录入的IP和PORT信息和数据库的当前实际信息是一致的"),
ORACLE_CONN_DB_ERROR("ORACLEErrCode-03","数据库名称错误，请检查数据库实例名称或者联系DBA确认该实例是否存在并且在正常服务"),
//execute query错误
MYSQL_QUERY_TABLE_NAME_ERROR("MYSQLErrCode-04","表不存在，请检查表名或者联系DBA确认该表是否存在"),
MYSQL_QUERY_SQL_ERROR("MYSQLErrCode-05","SQL语句执行出错，请检查Where条件是否存在拼写或语法错误"),
MYSQL_QUERY_COLUMN_ERROR("MYSQLErrCode-06","Column信息错误，请检查该列是否存在，如果是常量或者变量，请使用英文单引号’包起来"),
MYSQL_QUERY_SELECT_PRI_ERROR("MYSQLErrCode-07","读表数据出错，因为账号没有读表的权限，请联系DBA确认该账号的权限并授权"),
ORACLE_QUERY_TABLE_NAME_ERROR("ORACLEErrCode-04","表不存在，请检查表名或者联系DBA确认该表是否存在"),
ORACLE_QUERY_SQL_ERROR("ORACLEErrCode-05","SQL语句执行出错，原因可能是你填写的列不存在或者where条件不符合要求，1，请检查该列是否存在，如果是常量或者变量，请使用英文单引号’包起来; 2，请检查Where条件是否存在拼写或语法错误"),
ORACLE_QUERY_SELECT_PRI_ERROR("ORACLEErrCode-06","读表数据出错，因为账号没有读表的权限，请联系DBA确认该账号的权限并授权"),
ORACLE_QUERY_SQL_PARSER_ERROR("ORACLEErrCode-07","SQL语法出错，请检查Where条件是否存在拼写或语法错误"),
//PreSql,Post Sql错误
MYSQL_PRE_SQL_ERROR("MYSQLErrCode-08","PreSQL语法错误，请检查"),
MYSQL_POST_SQL_ERROR("MYSQLErrCode-09","PostSql语法错误，请检查"),
MYSQL_QUERY_SQL_PARSER_ERROR("MYSQLErrCode-10","SQL语法出错，请检查Where条件是否存在拼写或语法错误"),
ORACLE_PRE_SQL_ERROR("ORACLEErrCode-08", "PreSQL语法错误，请检查"),
ORACLE_POST_SQL_ERROR("ORACLEErrCode-09", "PostSql语法错误，请检查"),
//SplitPK 错误
MYSQL_SPLIT_PK_ERROR("MYSQLErrCode-11","SplitPK错误，请检查"),
ORACLE_SPLIT_PK_ERROR("ORACLEErrCode-10","SplitPK错误，请检查"),
//Insert,Delete 权限错误
MYSQL_INSERT_ERROR("MYSQLErrCode-12","数据库没有写权限，请联系DBA"),
MYSQL_DELETE_ERROR("MYSQLErrCode-13","数据库没有Delete权限，请联系DBA"),
ORACLE_INSERT_ERROR("ORACLEErrCode-11","数据库没有写权限，请联系DBA"),
ORACLE_DELETE_ERROR("ORACLEErrCode-12","数据库没有Delete权限，请联系DBA"),
JDBC_NULL("DBUtilErrorCode-20","JDBC URL为空，请检查配置"),
JDBC_OB10_ADDRESS_ERROR("DBUtilErrorCode-OB10-01","JDBC OB10格式错误，请联系askdatax"),
CONF_ERROR("DBUtilErrorCode-00", "您的配置错误."),
CONN_DB_ERROR("DBUtilErrorCode-10", "连接数据库失败. 请检查您的 账号、密码、数据库名称、IP、Port或者向 DBA 寻求帮助(注意网络环境)."),
GET_COLUMN_INFO_FAILED("DBUtilErrorCode-01", "获取表字段相关信息失败."),
UNSUPPORTED_TYPE("DBUtilErrorCode-12", "不支持的数据库类型. 请注意查看 DataX 已经支持的数据库类型以及数据库版本."),
COLUMN_SPLIT_ERROR("DBUtilErrorCode-13", "根据主键进行切分失败."),
SET_SESSION_ERROR("DBUtilErrorCode-14", "设置 session 失败."),
RS_ASYNC_ERROR("DBUtilErrorCode-15", "异步获取ResultSet next失败."),
REQUIRED_VALUE("DBUtilErrorCode-03", "您缺失了必须填写的参数值."),
ILLEGAL_VALUE("DBUtilErrorCode-02", "您填写的参数值不合法."),
ILLEGAL_SPLIT_PK("DBUtilErrorCode-04", "您填写的主键列不合法, DataX 仅支持切分主键为一个,并且类型为整数或者字符串类型."),
SPLIT_FAILED_ILLEGAL_SQL("DBUtilErrorCode-15", "DataX尝试切分表时, 执行数据库 Sql 失败. 请检查您的配置 table/splitPk/where 并作出修改."),
SQL_EXECUTE_FAIL("DBUtilErrorCode-06", "执行数据库 Sql 失败, 请检查您的配置的 column/table/where/querySql或者向 DBA 寻求帮助."),
// only for reader
READ_RECORD_FAIL("DBUtilErrorCode-07", "读取数据库数据失败. 请检查您的配置的 column/table/where/querySql或者向 DBA 寻求帮助."),
TABLE_QUERYSQL_MIXED("DBUtilErrorCode-08", "您配置凌乱了. 不能同时既配置table又配置querySql"),
TABLE_QUERYSQL_MISSING("DBUtilErrorCode-09", "您配置错误. table和querySql 应该并且只能配置一个."),
// only for writer
WRITE_DATA_ERROR("DBUtilErrorCode-05", "往您配置的写入表中写入数据时失败."),
NO_INSERT_PRIVILEGE("DBUtilErrorCode-11", "数据库没有写权限，请联系DBA"),
NO_DELETE_PRIVILEGE("DBUtilErrorCode-16", "数据库没有DELETE权限，请联系DBA"),
;
private final String code;
private final String description;
private DBUtilErrorCode(String code, String description) {
this.code = code;
this.description = description;
}
@Override
public String getCode() {
return this.code;
}
@Override
public String getDescription() {
return this.description;
}
@Override
public String toString() {
return String.format("Code:[%s], Description:[%s]. ", this.code,
this.description);
}
}
CommonErrorCode
/**
*
*/
public enum CommonErrorCode implements ErrorCode {
CONFIG_ERROR("Common-00", "您提供的配置文件存在错误信息，请检查您的作业配置 ."),
CONVERT_NOT_SUPPORT("Common-01", "同步数据出现业务脏数据情况，数据类型转换错误 ."),
CONVERT_OVER_FLOW("Common-02", "同步数据出现业务脏数据情况，数据类型转换溢出 ."),
RETRY_FAIL("Common-10", "方法调用多次仍旧失败 ."),
RUNTIME_ERROR("Common-11", "运行时内部调用错误 ."),
HOOK_INTERNAL_ERROR("Common-12", "Hook运行错误 ."),
SHUT_DOWN_TASK("Common-20", "Task收到了shutdown指令，为failover做准备"),
WAIT_TIME_EXCEED("Common-21", "等待时间超出范围"),
TASK_HUNG_EXPIRED("Common-22", "任务hung住，Expired");
private final String code;
private final String describe;
private CommonErrorCode(String code, String describe) {
this.code = code;
this.describe = describe;
}
@Override
public String getCode() {
return this.code;
}
@Override
public String getDescription() {
return this.describe;
}
@Override
public String toString() {
return String.format("Code:[%s], Describe:[%s]", this.code,
this.describe);
}
}
Configuration
package cn.com.bluemoon.bdscheduler.integration.util.db;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.CharUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.ToStringBuilder;
import java.io.*;
import java.util.*;
/**
* Configuration 提供多级JSON配置信息无损存储 <br>
* <br>
* <p/>
* 实例代码:<br>
* <p/>
* 获取job的配置信息<br>
* Configuration configuration = Configuration.from(new File("Config.json")); <br>
* String jobContainerClass =
* configuration.getString("core.container.job.class"); <br>
* <p/>
* <br>
* 设置多级List <br>
* configuration.set("job.reader.parameter.jdbcUrl", Arrays.asList(new String[]
* {"jdbc", "jdbc"}));
* <p/>
* <p/>
* <br>
* <br>
* 合并Configuration: <br>
* configuration.merge(another);
* <p/>
* <p/>
* <br>
* <br>
* <br>
* <p/>
* Configuration 存在两种较好地实现方式<br>
* 第一种是将JSON配置信息中所有的Key全部打平，用a.b.c的级联方式作为Map的Key，内部使用一个Map保存信息 <br>
* 第二种是将JSON的对象直接使用结构化树形结构保存<br>
* <p/>
* 目前使用的第二种实现方式，使用第一种的问题在于: <br>
* 1. 插入新对象，比较难处理，例如a.b.c="bazhen"，此时如果需要插入a="bazhen"，也即是根目录下第一层所有类型全部要废弃
* ，使用"bazhen"作为value，第一种方式使用字符串表示key，难以处理这类问题。 <br>
* 2. 返回树形结构，例如 a.b.c.d = "bazhen"，如果返回"a"下的所有元素，实际上是一个Map，需要合并处理 <br>
* 3. 输出JSON，将上述对象转为JSON，要把上述Map的多级key转为树形结构，并输出为JSON <br>
*/
public class Configuration {
/**
* 对于加密的keyPath，需要记录下来
* 为的是后面分布式情况下将该值加密后抛到DataXServer中
*/
private Set<String> secretKeyPathSet =
new HashSet<String>();
private Object root = null;
/**
* 初始化空白的Configuration
*/
public static Configuration newDefault() {
return Configuration.from("{}");
}
/**
* 从JSON字符串加载Configuration
*/
public static Configuration from(String json) {
json = StrUtil.replaceVariable(json);
checkJSON(json);
try {
return new Configuration(json);
} catch (Exception e) {
throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
e);
}
}
/**
* 从包括json的File对象加载Configuration
*/
public static Configuration from(File file) {
try {
return Configuration.from(IOUtils
.toString(new FileInputStream(file)));
} catch (FileNotFoundException e) {
throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
String.format("配置信息错误，您提供的配置文件[%s]不存在. 请检查您的配置文件.", file.getAbsolutePath()));
} catch (IOException e) {
throw DataXException.asDataXException(
CommonErrorCode.CONFIG_ERROR,
String.format("配置信息错误. 您提供配置文件[%s]读取失败，错误原因: %s. 请检查您的配置文件的权限设置.",
file.getAbsolutePath(), e));
}
}
/**
* 从包括json的InputStream对象加载Configuration
*/
public static Configuration from(InputStream is) {
try {
return Configuration.from(IOUtils.toString(is));
} catch (IOException e) {
throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
String.format("请检查您的配置文件. 您提供的配置文件读取失败，错误原因: %s. 请检查您的配置文件的权限设置.", e));
}
}
/**
* 从Map对象加载Configuration
*/
public static Configuration from(final Map<String, Object> object) {
return Configuration.from(Configuration.toJSONString(object));
}
/**
* 从List对象加载Configuration
*/
public static Configuration from(final List<Object> object) {
return Configuration.from(Configuration.toJSONString(object));
}
public String getNecessaryValue(String key, ErrorCode errorCode) {
String value = this.getString(key, null);
if (StringUtils.isBlank(value)) {
throw DataXException.asDataXException(errorCode,
String.format("您提供配置文件有误，[%s]是必填参数，不允许为空或者留白 .", key));
}
return value;
}
public String getUnnecessaryValue(String key,String defaultValue,ErrorCode errorCode) {
String value = this.getString(key, defaultValue);
if (StringUtils.isBlank(value)) {
value = defaultValue;
}
return value;
}
public Boolean getNecessaryBool(String key, ErrorCode errorCode) {
Boolean value = this.getBool(key);
if (value == null) {
throw DataXException.asDataXException(errorCode,
String.format("您提供配置文件有误，[%s]是必填参数，不允许为空或者留白 .", key));
}
return value;
}
/**
* 根据用户提供的json path，寻址具体的对象。
* <p/>
* <br>
* <p/>
* NOTE: 目前仅支持Map以及List下标寻址, 例如:
* <p/>
* <br />
* <p/>
* 对于如下JSON
* <p/>
* {"a": {"b": {"c": [0,1,2,3]}}}
* <p/>
* config.get("") 返回整个Map <br>
* config.get("a") 返回a下属整个Map <br>
* config.get("a.b.c") 返回c对应的数组List <br>
* config.get("a.b.c[0]") 返回数字0
*
* @return Java表示的JSON对象，如果path不存在或者对象不存在，均返回null。
*/
public Object get(final String path) {
this.checkPath(path);
try {
return this.findObject(path);
} catch (Exception e) {
return null;
}
}
/**
* 用户指定部分path，获取Configuration的子集
* <p/>
* <br>
* 如果path获取的路径或者对象不存在，返回null
*/
public Configuration getConfiguration(final String path) {
Object object = this.get(path);
if (null == object) {
return null;
}
return Configuration.from(Configuration.toJSONString(object));
}
/**
* 根据用户提供的json path，寻址String对象
*
* @return String对象，如果path不存在或者String不存在，返回null
*/
public String getString(final String path) {
Object string = this.get(path);
if (null == string) {
return null;
}
return String.valueOf(string);
}
/**
* 根据用户提供的json path，寻址String对象，如果对象不存在，返回默认字符串
*
* @return String对象，如果path不存在或者String不存在，返回默认字符串
*/
public String getString(final String path, final String defaultValue) {
String result = this.getString(path);
if (null == result) {
return defaultValue;
}
return result;
}
/**
* 根据用户提供的json path，寻址Character对象
*
* @return Character对象，如果path不存在或者Character不存在，返回null
*/
public Character getChar(final String path) {
String result = this.getString(path);
if (null == result) {
return null;
}
try {
return CharUtils.toChar(result);
} catch (Exception e) {
throw DataXException.asDataXException(
CommonErrorCode.CONFIG_ERROR,
String.format("任务读取配置文件出错. 因为配置文件路径[%s] 值非法，期望是字符类型: %s. 请检查您的配置并作出修改.", path,
e.getMessage()));
}
}
/**
* 根据用户提供的json path，寻址Boolean对象，如果对象不存在，返回默认Character对象
*
* @return Character对象，如果path不存在或者Character不存在，返回默认Character对象
*/
public Character getChar(final String path, char defaultValue) {
Character result = this.getChar(path);
if (null == result) {
return defaultValue;
}
return result;
}
/**
* 根据用户提供的json path，寻址Boolean对象
*
* @return Boolean对象，如果path值非true,false ，将报错.特别注意：当 path 不存在时，会返回：null.
*/
public Boolean getBool(final String path) {
String result = this.getString(path);
if (null == result) {
return null;
} else if ("true".equalsIgnoreCase(result)) {
return Boolean.TRUE;
} else if ("false".equalsIgnoreCase(result)) {
return Boolean.FALSE;
} else {
throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
String.format("您提供的配置信息有误，因为从[%s]获取的值[%s]无法转换为bool类型. 请检查源表的配置并且做出相应的修改.",
path, result));
}
}
/**
* 根据用户提供的json path，寻址Boolean对象，如果对象不存在，返回默认Boolean对象
*
* @return Boolean对象，如果path不存在或者Boolean不存在，返回默认Boolean对象
*/
public Boolean getBool(final String path, boolean defaultValue) {
Boolean result = this.getBool(path);
if (null == result) {
return defaultValue;
}
return result;
}
/**
* 根据用户提供的json path，寻址Integer对象
*
* @return Integer对象，如果path不存在或者Integer不存在，返回null
*/
public Integer getInt(final String path) {
String result = this.getString(path);
if (null == result) {
return null;
}
try {
return Integer.valueOf(result);
} catch (Exception e) {
throw DataXException.asDataXException(
CommonErrorCode.CONFIG_ERROR,
String.format("任务读取配置文件出错. 配置文件路径[%s] 值非法, 期望是整数类型: %s. 请检查您的配置并作出修改.", path,
e.getMessage()));
}
}
/**
* 根据用户提供的json path，寻址Integer对象，如果对象不存在，返回默认Integer对象
*
* @return Integer对象，如果path不存在或者Integer不存在，返回默认Integer对象
*/
public Integer getInt(final String path, int defaultValue) {
Integer object = this.getInt(path);
if (null == object) {
return defaultValue;
}
return object;
}
/**
* 根据用户提供的json path，寻址Long对象
*
* @return Long对象，如果path不存在或者Long不存在，返回null
*/
public Long getLong(final String path) {
String result = this.getString(path);
if (StringUtils.isBlank(result)) {
return null;
}
try {
return Long.valueOf(result);
} catch (Exception e) {
throw DataXException.asDataXException(
CommonErrorCode.CONFIG_ERROR,
String.format("任务读取配置文件出错. 配置文件路径[%s] 值非法, 期望是整数类型: %s. 请检查您的配置并作出修改.", path,
e.getMessage()));
}
}
/**
* 根据用户提供的json path，寻址Long对象，如果对象不存在，返回默认Long对象
*
* @return Long对象，如果path不存在或者Integer不存在，返回默认Long对象
*/
public Long getLong(final String path, long defaultValue) {
Long result = this.getLong(path);
if (null == result) {
return defaultValue;
}
return result;
}
/**
* 根据用户提供的json path，寻址Double对象
*
* @return Double对象，如果path不存在或者Double不存在，返回null
*/
public Double getDouble(final String path) {
String result = this.getString(path);
if (StringUtils.isBlank(result)) {
return null;
}
try {
return Double.valueOf(result);
} catch (Exception e) {
throw DataXException.asDataXException(
CommonErrorCode.CONFIG_ERROR,
String.format("任务读取配置文件出错. 配置文件路径[%s] 值非法, 期望是浮点类型: %s. 请检查您的配置并作出修改.", path,
e.getMessage()));
}
}
/**
* 根据用户提供的json path，寻址Double对象，如果对象不存在，返回默认Double对象
*
* @return Double对象，如果path不存在或者Double不存在，返回默认Double对象
*/
public Double getDouble(final String path, double defaultValue) {
Double result = this.getDouble(path);
if (null == result) {
return defaultValue;
}
return result;
}
/**
* 根据用户提供的json path，寻址List对象，如果对象不存在，返回null
*/
@SuppressWarnings("unchecked")
public List<Object> getList(final String path) {
List<Object> list = this.get(path, List.class);
if (null == list) {
return null;
}
return list;
}
/**
* 根据用户提供的json path，寻址List对象，如果对象不存在，返回null
*/
@SuppressWarnings("unchecked")
public <T> List<T> getList(final String path, Class<T> t) {
Object object = this.get(path, List.class);
if (null == object) {
return null;
}
List<T> result = new ArrayList<T>();
List<Object> origin = (List<Object>) object;
for (final Object each : origin) {
result.add((T) each);
}
return result;
}
/**
* 根据用户提供的json path，寻址List对象，如果对象不存在，返回默认List
*/
@SuppressWarnings("unchecked")
public List<Object> getList(final String path,
final List<Object> defaultList) {
Object object = this.getList(path);
if (null == object) {
return defaultList;
}
return (List<Object>) object;
}
/**
* 根据用户提供的json path，寻址List对象，如果对象不存在，返回默认List
*/
public <T> List<T> getList(final String path, final List<T> defaultList,
Class<T> t) {
List<T> list = this.getList(path, t);
if (null == list) {
return defaultList;
}
return list;
}
/**
* 根据用户提供的json path，寻址包含Configuration的List，如果对象不存在，返回默认null
*/
public List<Configuration> getListConfiguration(final String path) {
List<Object> lists = getList(path);
if (lists == null) {
return null;
}
List<Configuration> result = new ArrayList<Configuration>();
for (final Object object : lists) {
result.add(Configuration.from(Configuration.toJSONString(object)));
}
return result;
}
/**
* 根据用户提供的json path，寻址Map对象，如果对象不存在，返回null
*/
@SuppressWarnings("unchecked")
public Map<String, Object> getMap(final String path) {
Map<String, Object> result = this.get(path, Map.class);
if (null == result) {
return null;
}
return result;
}
/**
* 根据用户提供的json path，寻址Map对象，如果对象不存在，返回null;
*/
@SuppressWarnings("unchecked")
public <T> Map<String, T> getMap(final String path, Class<T> t) {
Map<String, Object> map = this.get(path, Map.class);
if (null == map) {
return null;
}
Map<String, T> result = new HashMap<String, T>();
for (final String key : map.keySet()) {
result.put(key, (T) map.get(key));
}
return result;
}
/**
* 根据用户提供的json path，寻址Map对象，如果对象不存在，返回默认map
*/
@SuppressWarnings("unchecked")
public Map<String, Object> getMap(final String path,
final Map<String, Object> defaultMap) {
Object object = this.getMap(path);
if (null == object) {
return defaultMap;
}
return (Map<String, Object>) object;
}
/**
* 根据用户提供的json path，寻址Map对象，如果对象不存在，返回默认map
*/
public <T> Map<String, T> getMap(final String path,
final Map<String, T> defaultMap, Class<T> t) {
Map<String, T> result = getMap(path, t);
if (null == result) {
return defaultMap;
}
return result;
}
/**
* 根据用户提供的json path，寻址包含Configuration的Map，如果对象不存在，返回默认null
*/
@SuppressWarnings("unchecked")
public Map<String, Configuration> getMapConfiguration(final String path) {
Map<String, Object> map = this.get(path, Map.class);
if (null == map) {
return null;
}
Map<String, Configuration> result = new HashMap<String, Configuration>();
for (final String key : map.keySet()) {
result.put(key, Configuration.from(Configuration.toJSONString(map
.get(key))));
}
return result;
}
/**
* 根据用户提供的json path，寻址具体的对象，并转为用户提供的类型
* <p/>
* <br>
* <p/>
* NOTE: 目前仅支持Map以及List下标寻址, 例如:
* <p/>
* <br />
* <p/>
* 对于如下JSON
* <p/>
* {"a": {"b": {"c": [0,1,2,3]}}}
* <p/>
* config.get("") 返回整个Map <br>
* config.get("a") 返回a下属整个Map <br>
* config.get("a.b.c") 返回c对应的数组List <br>
* config.get("a.b.c[0]") 返回数字0
*
* @return Java表示的JSON对象，如果转型失败，将抛出异常
*/
@SuppressWarnings("unchecked")
public <T> T get(final String path, Class<T> clazz) {
this.checkPath(path);
return (T) this.get(path);
}
/**
* 格式化Configuration输出
*/
public String beautify() {
return JSON.toJSONString(this.getInternal(),
SerializerFeature.PrettyFormat);
}
/**
* 根据用户提供的json path，插入指定对象，并返回之前存在的对象(如果存在)
* <p/>
* <br>
* <p/>
* 目前仅支持.以及数组下标寻址, 例如:
* <p/>
* <br />
* <p/>
* config.set("a.b.c[3]", object);
* <p/>
* <br>
* 对于插入对象，Configuration不做任何限制，但是请务必保证该对象是简单对象(包括Map<String,
* Object>、List<Object>)，不要使用自定义对象，否则后续对于JSON序列化等情况会出现未定义行为。
*
* @param path
* JSON path对象
* @param object
* 需要插入的对象
* @return Java表示的JSON对象
*/
public Object set(final String path, final Object object) {
checkPath(path);
Object result = this.get(path);
setObject(path, extractConfiguration(object));
return result;
}
/**
* 获取Configuration下所有叶子节点的key
* <p/>
* <br>
* <p/>
* 对于<br>
* <p/>
* {"a": {"b": {"c": [0,1,2,3]}}, "x": "y"}
* <p/>
* 下属的key包括: a.b.c[0],a.b.c[1],a.b.c[2],a.b.c[3],x
*/
public Set<String> getKeys() {
Set<String> collect = new HashSet<String>();
this.getKeysRecursive(this.getInternal(), "", collect);
return collect;
}
/**
* 删除path对应的值，如果path不存在，将抛出异常。
*/
public Object remove(final String path) {
final Object result = this.get(path);
if (null == result) {
throw DataXException.asDataXException(
CommonErrorCode.RUNTIME_ERROR,
String.format("配置文件对应Key[%s]并不存在，该情况是代码编程错误. 请联系DataX团队的同学.", path));
}
this.set(path, null);
return result;
}
/**
* 合并其他Configuration，并修改两者冲突的KV配置
*
* @param another
* 合并加入的第三方Configuration
* @param updateWhenConflict
* 当合并双方出现KV冲突时候，选择更新当前KV，或者忽略该KV
* @return 返回合并后对象
*/
public Configuration merge(final Configuration another,
boolean updateWhenConflict) {
Set<String> keys = another.getKeys();
for (final String key : keys) {
// 如果使用更新策略，凡是another存在的key，均需要更新
if (updateWhenConflict) {
this.set(key, another.get(key));
continue;
}
// 使用忽略策略，只有another Configuration存在但是当前Configuration不存在的key，才需要更新
boolean isCurrentExists = this.get(key) != null;
if (isCurrentExists) {
continue;
}
this.set(key, another.get(key));
}
return this;
}
@Override
public String toString() {
return this.toJSON();
}
/**
* 将Configuration作为JSON输出
*/
public String toJSON() {
return Configuration.toJSONString(this.getInternal());
}
/**
* 拷贝当前Configuration，注意，这里使用了深拷贝，避免冲突
*/
public Configuration clone() {
Configuration config = Configuration
.from(Configuration.toJSONString(this.getInternal()));
config.addSecretKeyPath(this.secretKeyPathSet);
return config;
}
/**
* 按照configuration要求格式的path
* 比如：
* a.b.c
* a.b[2].c
* @param path
*/
public void addSecretKeyPath(String path) {
if(StringUtils.isNotBlank(path)) {
this.secretKeyPathSet.add(path);
}
}
public void addSecretKeyPath(Set<String> pathSet) {
if(pathSet != null) {
this.secretKeyPathSet.addAll(pathSet);
}
}
public void setSecretKeyPathSet(Set<String> keyPathSet) {
if(keyPathSet != null) {
this.secretKeyPathSet = keyPathSet;
}
}
public boolean isSecretPath(String path) {
return this.secretKeyPathSet.contains(path);
}
@SuppressWarnings("unchecked")
void getKeysRecursive(final Object current, String path, Set<String> collect) {
boolean isRegularElement = !(current instanceof Map || current instanceof List);
if (isRegularElement) {
collect.add(path);
return;
}
boolean isMap = current instanceof Map;
if (isMap) {
Map<String, Object> mapping = ((Map<String, Object>) current);
for (final String key : mapping.keySet()) {
if (StringUtils.isBlank(path)) {
getKeysRecursive(mapping.get(key), key.trim(), collect);
} else {
getKeysRecursive(mapping.get(key), path + "." + key.trim(),
collect);
}
}
return;
}
boolean isList = current instanceof List;
if (isList) {
List<Object> lists = (List<Object>) current;
for (int i = 0; i < lists.size(); i++) {
getKeysRecursive(lists.get(i), path + String.format("[%d]", i),
collect);
}
return;
}
return;
}
public Object getInternal() {
return this.root;
}
private void setObject(final String path, final Object object) {
Object newRoot = setObjectRecursive(this.root, split2List(path), 0,
object);
if (isSuitForRoot(newRoot)) {
this.root = newRoot;
return;
}
throw DataXException.asDataXException(CommonErrorCode.RUNTIME_ERROR,
String.format("值[%s]无法适配您提供[%s]， 该异常代表系统编程错误, 请联系DataX开发团队!",
ToStringBuilder.reflectionToString(object), path));
}
@SuppressWarnings("unchecked")
private Object extractConfiguration(final Object object) {
if (object instanceof Configuration) {
return extractFromConfiguration(object);
}
if (object instanceof List) {
List<Object> result = new ArrayList<Object>();
for (final Object each : (List<Object>) object) {
result.add(extractFromConfiguration(each));
}
return result;
}
if (object instanceof Map) {
Map<String, Object> result = new HashMap<String, Object>();
for (final String key : ((Map<String, Object>) object).keySet()) {
result.put(key,
extractFromConfiguration(((Map<String, Object>) object)
.get(key)));
}
return result;
}
return object;
}
private Object extractFromConfiguration(final Object object) {
if (object instanceof Configuration) {
return ((Configuration) object).getInternal();
}
return object;
}
Object buildObject(final List<String> paths, final Object object) {
if (null == paths) {
throw DataXException.asDataXException(
CommonErrorCode.RUNTIME_ERROR,
"Path不能为null，该异常代表系统编程错误, 请联系DataX开发团队 !");
}
if (1 == paths.size() && StringUtils.isBlank(paths.get(0))) {
return object;
}
Object child = object;
for (int i = paths.size() - 1; i >= 0; i--) {
String path = paths.get(i);
if (isPathMap(path)) {
Map<String, Object> mapping = new HashMap<String, Object>();
mapping.put(path, child);
child = mapping;
continue;
}
if (isPathList(path)) {
List<Object> lists = new ArrayList<Object>(
this.getIndex(path) + 1);
expand(lists, this.getIndex(path) + 1);
lists.set(this.getIndex(path), child);
child = lists;
continue;
}
throw DataXException.asDataXException(
CommonErrorCode.RUNTIME_ERROR, String.format(
"路径[%s]出现非法值类型[%s]，该异常代表系统编程错误, 请联系DataX开发团队! .",
StringUtils.join(paths, "."), path));
}
return child;
}
@SuppressWarnings("unchecked")
Object setObjectRecursive(Object current, final List<String> paths,
int index, final Object value) {
// 如果是已经超出path，我们就返回value即可，作为最底层叶子节点
boolean isLastIndex = index == paths.size();
if (isLastIndex) {
return value;
}
String path = paths.get(index).trim();
boolean isNeedMap = isPathMap(path);
if (isNeedMap) {
Map<String, Object> mapping;
// 当前不是map，因此全部替换为map，并返回新建的map对象
boolean isCurrentMap = current instanceof Map;
if (!isCurrentMap) {
mapping = new HashMap<String, Object>();
mapping.put(
path,
buildObject(paths.subList(index + 1, paths.size()),
value));
return mapping;
}
// 当前是map，但是没有对应的key，也就是我们需要新建对象插入该map，并返回该map
mapping = ((Map<String, Object>) current);
boolean hasSameKey = mapping.containsKey(path);
if (!hasSameKey) {
mapping.put(
path,
buildObject(paths.subList(index + 1, paths.size()),
value));
return mapping;
}
// 当前是map，而且还竟然存在这个值，好吧，继续递归遍历
current = mapping.get(path);
mapping.put(path,
setObjectRecursive(current, paths, index + 1, value));
return mapping;
}
boolean isNeedList = isPathList(path);
if (isNeedList) {
List<Object> lists;
int listIndexer = getIndex(path);
// 当前是list，直接新建并返回即可
boolean isCurrentList = current instanceof List;
if (!isCurrentList) {
lists = expand(new ArrayList<Object>(), listIndexer + 1);
lists.set(
listIndexer,
buildObject(paths.subList(index + 1, paths.size()),
value));
return lists;
}
// 当前是list，但是对应的indexer是没有具体的值，也就是我们新建对象然后插入到该list，并返回该List
lists = (List<Object>) current;
lists = expand(lists, listIndexer + 1);
boolean hasSameIndex = lists.get(listIndexer) != null;
if (!hasSameIndex) {
lists.set(
listIndexer,
buildObject(paths.subList(index + 1, paths.size()),
value));
return lists;
}
// 当前是list，并且存在对应的index，没有办法继续递归寻找
current = lists.get(listIndexer);
lists.set(listIndexer,
setObjectRecursive(current, paths, index + 1, value));
return lists;
}
throw DataXException.asDataXException(CommonErrorCode.RUNTIME_ERROR,
"该异常代表系统编程错误, 请联系DataX开发团队 !");
}
private Object findObject(final String path) {
boolean isRootQuery = StringUtils.isBlank(path);
if (isRootQuery) {
return this.root;
}
Object target = this.root;
for (final String each : split2List(path)) {
if (isPathMap(each)) {
target = findObjectInMap(target, each);
continue;
} else {
target = findObjectInList(target, each);
continue;
}
}
return target;
}
@SuppressWarnings("unchecked")
private Object findObjectInMap(final Object target, final String index) {
boolean isMap = (target instanceof Map);
if (!isMap) {
throw new IllegalArgumentException(String.format(
"您提供的配置文件有误. 路径[%s]需要配置Json格式的Map对象，但该节点发现实际类型是[%s]. 请检查您的配置并作出修改.",
index, target.getClass().toString()));
}
Object result = ((Map<String, Object>) target).get(index);
if (null == result) {
throw new IllegalArgumentException(String.format(
"您提供的配置文件有误. 路径[%s]值为null，datax无法识别该配置. 请检查您的配置并作出修改.", index));
}
return result;
}
@SuppressWarnings({ "unchecked" })
private Object findObjectInList(final Object target, final String each) {
boolean isList = (target instanceof List);
if (!isList) {
throw new IllegalArgumentException(String.format(
"您提供的配置文件有误. 路径[%s]需要配置Json格式的Map对象，但该节点发现实际类型是[%s]. 请检查您的配置并作出修改.",
each, target.getClass().toString()));
}
String index = each.replace("[", "").replace("]", "");
if (!StringUtils.isNumeric(index)) {
throw new IllegalArgumentException(
String.format(
"系统编程错误，列表下标必须为数字类型，但该节点发现实际类型是[%s] ，该异常代表系统编程错误, 请联系DataX开发团队 !",
index));
}
return ((List<Object>) target).get(Integer.valueOf(index));
}
private List<Object> expand(List<Object> list, int size) {
int expand = size - list.size();
while (expand-- > 0) {
list.add(null);
}
return list;
}
private boolean isPathList(final String path) {
return path.contains("[") && path.contains("]");
}
private boolean isPathMap(final String path) {
return StringUtils.isNotBlank(path) && !isPathList(path);
}
private int getIndex(final String index) {
return Integer.valueOf(index.replace("[", "").replace("]", ""));
}
private boolean isSuitForRoot(final Object object) {
if (null != object && (object instanceof List || object instanceof Map)) {
return true;
}
return false;
}
private String split(final String path) {
return StringUtils.replace(path, "[", ".[");
}
private List<String> split2List(final String path) {
return Arrays.asList(StringUtils.split(split(path), "."));
}
private void checkPath(final String path) {
if (null == path) {
throw new IllegalArgumentException(
"系统编程错误, 该异常代表系统编程错误, 请联系DataX开发团队!.");
}
for (final String each : StringUtils.split(".")) {
if (StringUtils.isBlank(each)) {
throw new IllegalArgumentException(String.format(
"系统编程错误, 路径[%s]不合法, 路径层次之间不能出现空白字符 .", path));
}
}
}
@SuppressWarnings("unused")
private String toJSONPath(final String path) {
return (StringUtils.isBlank(path) ? "$" : "$." + path).replace("$.[",
"$[");
}
private static void checkJSON(final String json) {
if (StringUtils.isBlank(json)) {
throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
"配置信息错误. 因为您提供的配置信息不是合法的JSON格式, JSON不能为空白. 请按照标准json格式提供配置信息. ");
}
}
private Configuration(final String json) {
try {
this.root = JSON.parse(json);
} catch (Exception e) {
throw DataXException.asDataXException(CommonErrorCode.CONFIG_ERROR,
String.format("配置信息错误. 您提供的配置信息不是合法的JSON格式: %s . 请按照标准json格式提供配置信息. ", e.getMessage()));
}
}
private static String toJSONString(final Object object) {
return JSON.toJSONString(object);
}
public Set<String> getSecretKeyPathSet() {
return secretKeyPathSet;
}
}
Constant
public final class Constant {
static final int TIMEOUT_SECONDS = 15;
static final int MAX_TRY_TIMES = 4;
static final int SOCKET_TIMEOUT_INSECOND = 172800;
public static final String MYSQL_DATABASE = "Unknown database";
public static final String MYSQL_CONNEXP = "Communications link failure";
public static final String MYSQL_ACCDENIED = "Access denied";
public static final String MYSQL_TABLE_NAME_ERR1 = "Table";
public static final String MYSQL_TABLE_NAME_ERR2 = "doesn't exist";
public static final String MYSQL_SELECT_PRI = "SELECT command denied to user";
public static final String MYSQL_COLUMN1 = "Unknown column";
public static final String MYSQL_COLUMN2 = "field list";
public static final String MYSQL_WHERE = "where clause";
public static final String ORACLE_DATABASE = "ORA-12505";
public static final String ORACLE_CONNEXP = "The Network Adapter could not establish the connection";
public static final String ORACLE_ACCDENIED = "ORA-01017";
public static final String ORACLE_TABLE_NAME = "table or view does not exist";
public static final String ORACLE_SELECT_PRI = "insufficient privileges";
public static final String ORACLE_SQL = "invalid identifier";
}
DataBaseType
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* refer:http://blog.csdn.net/ring0hx/article/details/6152528
* <p/>
*/
public enum DataBaseType {
MySql("mysql", "com.mysql.jdbc.Driver"),
Tddl("mysql", "com.mysql.jdbc.Driver"),
DRDS("drds", "com.mysql.jdbc.Driver"),
Oracle("oracle", "oracle.jdbc.OracleDriver"),
SQLServer("sqlserver", "com.microsoft.sqlserver.jdbc.SQLServerDriver"),
PostgreSQL("postgresql", "org.postgresql.Driver"),
RDBMS("rdbms", "com.alibaba.datax.plugin.rdbms.util.DataBaseType"),
DB2("db2", "com.ibm.db2.jcc.DB2Driver"),
ADS("ads","com.mysql.jdbc.Driver"),
ClickHouse("clickhouse", "ru.yandex.clickhouse.ClickHouseDriver"),
KingbaseES("kingbasees", "com.kingbase8.Driver"),
Oscar("oscar", "com.oscar.Driver"),
OceanBase("oceanbase", "com.alipay.oceanbase.jdbc.Driver");
private String typeName;
private String driverClassName;
DataBaseType(String typeName, String driverClassName) {
this.typeName = typeName;
this.driverClassName = driverClassName;
}
public String getDriverClassName() {
return this.driverClassName;
}
public String appendJDBCSuffixForReader(String jdbc) {
String result = jdbc;
String suffix = null;
switch (this) {
case MySql:
case DRDS:
case OceanBase:
suffix = "yearIsDateType=false&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false&rewriteBatchedStatements=true";
if (jdbc.contains("?")) {
result = jdbc + "&" + suffix;
} else {
result = jdbc + "?" + suffix;
}
break;
case Oracle:
break;
case SQLServer:
break;
case DB2:
break;
case PostgreSQL:
break;
case ClickHouse:
break;
case RDBMS:
break;
case KingbaseES:
break;
case Oscar:
break;
default:
throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type.");
}
return result;
}
public String appendJDBCSuffixForWriter(String jdbc) {
String result = jdbc;
String suffix = null;
switch (this) {
case MySql:
suffix = "yearIsDateType=false&zeroDateTimeBehavior=convertToNull&rewriteBatchedStatements=true&tinyInt1isBit=false";
if (jdbc.contains("?")) {
result = jdbc + "&" + suffix;
} else {
result = jdbc + "?" + suffix;
}
break;
case DRDS:
suffix = "yearIsDateType=false&zeroDateTimeBehavior=convertToNull";
if (jdbc.contains("?")) {
result = jdbc + "&" + suffix;
} else {
result = jdbc + "?" + suffix;
}
break;
case Oracle:
break;
case SQLServer:
break;
case DB2:
break;
case PostgreSQL:
break;
case ClickHouse:
break;
case RDBMS:
break;
case KingbaseES:
break;
case Oscar:
break;
case OceanBase:
suffix = "yearIsDateType=false&zeroDateTimeBehavior=convertToNull&tinyInt1isBit=false&rewriteBatchedStatements=true";
if (jdbc.contains("?")) {
result = jdbc + "&" + suffix;
} else {
result = jdbc + "?" + suffix;
}
break;
default:
throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type.");
}
return result;
}
public String formatPk(String splitPk) {
String result = splitPk;
switch (this) {
case MySql:
case Oracle:
if (splitPk.length() >= 2 && splitPk.startsWith("`") && splitPk.endsWith("`")) {
result = splitPk.substring(1, splitPk.length() - 1).toLowerCase();
}
break;
case SQLServer:
if (splitPk.length() >= 2 && splitPk.startsWith("[") && splitPk.endsWith("]")) {
result = splitPk.substring(1, splitPk.length() - 1).toLowerCase();
}
break;
case DB2:
case PostgreSQL:
case KingbaseES:
case Oscar:
break;
default:
throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type.");
}
return result;
}
public String quoteColumnName(String columnName) {
String result = columnName;
switch (this) {
case MySql:
result = "`" + columnName.replace("`", "``") + "`";
break;
case Oracle:
break;
case SQLServer:
result = "[" + columnName + "]";
break;
case DB2:
case PostgreSQL:
case KingbaseES:
case Oscar:
break;
default:
throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type");
}
return result;
}
public String quoteTableName(String tableName) {
String result = tableName;
switch (this) {
case MySql:
result = "`" + tableName.replace("`", "``") + "`";
break;
case Oracle:
break;
case SQLServer:
break;
case DB2:
break;
case PostgreSQL:
break;
case KingbaseES:
break;
case Oscar:
break;
default:
throw DataXException.asDataXException(DBUtilErrorCode.UNSUPPORTED_TYPE, "unsupported database type");
}
return result;
}
private static Pattern mysqlPattern = Pattern.compile("jdbc:mysql://(.+):\\d+/.+");
private static Pattern oraclePattern = Pattern.compile("jdbc:oracle:thin:@(.+):\\d+:.+");
/**
* 注意：目前只实现了从 mysql/oracle 中识别出ip 信息.未识别到则返回 null.
*/
public static String parseIpFromJdbcUrl(String jdbcUrl) {
Matcher mysql = mysqlPattern.matcher(jdbcUrl);
if (mysql.matches()) {
return mysql.group(1);
}
Matcher oracle = oraclePattern.matcher(jdbcUrl);
if (oracle.matches()) {
return oracle.group(1);
}
return null;
}
public String getTypeName() {
return typeName;
}
public void setTypeName(String typeName) {
this.typeName = typeName;
}
}
DataXException
import java.io.PrintWriter;
import java.io.StringWriter;
public class DataXException extends RuntimeException {
private static final long serialVersionUID = 1L;
private ErrorCode errorCode;
public DataXException(ErrorCode errorCode, String errorMessage) {
super(errorCode.toString() + " - " + errorMessage);
this.errorCode = errorCode;
}
public DataXException(String errorMessage) {
super(errorMessage);
}
private DataXException(ErrorCode errorCode, String errorMessage, Throwable cause) {
super(errorCode.toString() + " - " + getMessage(errorMessage) + " - " + getMessage(cause), cause);
this.errorCode = errorCode;
}
public static DataXException asDataXException(ErrorCode errorCode, String message) {
return new DataXException(errorCode, message);
}
public static DataXException asDataXException(String message) {
return new DataXException(message);
}
public static DataXException asDataXException(ErrorCode errorCode, String message, Throwable cause) {
if (cause instanceof DataXException) {
return (DataXException) cause;
}
return new DataXException(errorCode, message, cause);
}
public static DataXException asDataXException(ErrorCode errorCode, Throwable cause) {
if (cause instanceof DataXException) {
return (DataXException) cause;
}
return new DataXException(errorCode, getMessage(cause), cause);
}
public ErrorCode getErrorCode() {
return this.errorCode;
}
private static String getMessage(Object obj) {
if (obj == null) {
return "";
}
if (obj instanceof Throwable) {
StringWriter str = new StringWriter();
PrintWriter pw = new PrintWriter(str);
((Throwable) obj).printStackTrace(pw);
return str.toString();
// return ((Throwable) obj).getMessage();
} else {
return obj.toString();
}
}
}
ErrorCode
/**
* 尤其注意：最好提供toString()实现。例如：
*
* <pre>
*
* @Override
* public String toString() {
* return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
* }
* </pre>
*
*/
public interface ErrorCode {
// 错误码编号
String getCode();
// 错误码描述
String getDescription();
/** 必须提供toString的实现
*
* <pre>
* @Override
* public String toString() {
* return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
* }
* </pre>
*
*/
String toString();
}
Key
/**
* 编码，时区等配置，暂未定.
*/
public final class Key {
public final static String JDBC_URL = "jdbcUrl";
public final static String USERNAME = "username";
public final static String PASSWORD = "password";
public final static String TABLE = "table";
public final static String MANDATORY_ENCODING = "mandatoryEncoding";
// 是数组配置
public final static String COLUMN = "column";
public final static String COLUMN_LIST = "columnList";
public final static String WHERE = "where";
public final static String HINT = "hint";
public final static String SPLIT_PK = "splitPk";
public final static String SPLIT_MODE = "splitMode";
public final static String SAMPLE_PERCENTAGE = "samplePercentage";
public final static String QUERY_SQL = "querySql";
public final static String SPLIT_PK_SQL = "splitPkSql";
public final static String PRE_SQL = "preSql";
public final static String POST_SQL = "postSql";
public final static String CHECK_SLAVE = "checkSlave";
public final static String SESSION = "session";
public final static String DBNAME = "dbName";
public final static String DRYRUN = "dryRun";
public static String SPLIT_FACTOR = "splitFactor";
public final static String WEAK_READ = "weakRead";
public final static String SAVE_POINT = "savePoint";
public final static String REUSE_CONN = "reuseConn";
public final static String PARTITION_NAME = "partitionName";
}
RdbmsException
/**
* Created by judy.lt on 2015/6/5.
*/
public class RdbmsException extends DataXException{
public RdbmsException(ErrorCode errorCode, String message){
super(errorCode,message);
}
public static DataXException asConnException(DataBaseType dataBaseType,Exception e,String userName,String dbName){
if (dataBaseType.equals(DataBaseType.MySql)){
DBUtilErrorCode dbUtilErrorCode = mySqlConnectionErrorAna(e.getMessage());
if (dbUtilErrorCode == DBUtilErrorCode.MYSQL_CONN_DB_ERROR && dbName !=null ){
return DataXException.asDataXException(dbUtilErrorCode,"该数据库名称为："+dbName+" 具体错误信息为："+e);
}
if (dbUtilErrorCode == DBUtilErrorCode.MYSQL_CONN_USERPWD_ERROR ){
return DataXException.asDataXException(dbUtilErrorCode,"该数据库用户名为："+userName+" 具体错误信息为："+e);
}
return DataXException.asDataXException(dbUtilErrorCode," 具体错误信息为："+e);
}
if (dataBaseType.equals(DataBaseType.Oracle)){
DBUtilErrorCode dbUtilErrorCode = oracleConnectionErrorAna(e.getMessage());
if (dbUtilErrorCode == DBUtilErrorCode.ORACLE_CONN_DB_ERROR && dbName != null){
return DataXException.asDataXException(dbUtilErrorCode,"该数据库名称为："+dbName+" 具体错误信息为："+e);
}
if (dbUtilErrorCode == DBUtilErrorCode.ORACLE_CONN_USERPWD_ERROR ){
return DataXException.asDataXException(dbUtilErrorCode,"该数据库用户名为："+userName+" 具体错误信息为："+e);
}
return DataXException.asDataXException(dbUtilErrorCode," 具体错误信息为："+e);
}
return DataXException.asDataXException(DBUtilErrorCode.CONN_DB_ERROR," 具体错误信息为："+e);
}
public static DBUtilErrorCode mySqlConnectionErrorAna(String e){
if (e.contains(Constant.MYSQL_DATABASE)){
return DBUtilErrorCode.MYSQL_CONN_DB_ERROR;
}
if (e.contains(Constant.MYSQL_CONNEXP)){
return DBUtilErrorCode.MYSQL_CONN_IPPORT_ERROR;
}
if (e.contains(Constant.MYSQL_ACCDENIED)){
return DBUtilErrorCode.MYSQL_CONN_USERPWD_ERROR;
}
return DBUtilErrorCode.CONN_DB_ERROR;
}
public static DBUtilErrorCode oracleConnectionErrorAna(String e){
if (e.contains(Constant.ORACLE_DATABASE)){
return DBUtilErrorCode.ORACLE_CONN_DB_ERROR;
}
if (e.contains(Constant.ORACLE_CONNEXP)){
return DBUtilErrorCode.ORACLE_CONN_IPPORT_ERROR;
}
if (e.contains(Constant.ORACLE_ACCDENIED)){
return DBUtilErrorCode.ORACLE_CONN_USERPWD_ERROR;
}
return DBUtilErrorCode.CONN_DB_ERROR;
}
public static DataXException asQueryException(DataBaseType dataBaseType, Exception e,String querySql,String table,String userName){
if (dataBaseType.equals(DataBaseType.MySql)){
DBUtilErrorCode dbUtilErrorCode = mySqlQueryErrorAna(e.getMessage());
if (dbUtilErrorCode == DBUtilErrorCode.MYSQL_QUERY_TABLE_NAME_ERROR && table != null){
return DataXException.asDataXException(dbUtilErrorCode,"表名为："+table+" 执行的SQL为:"+querySql+" 具体错误信息为："+e);
}
if (dbUtilErrorCode == DBUtilErrorCode.MYSQL_QUERY_SELECT_PRI_ERROR && userName != null){
return DataXException.asDataXException(dbUtilErrorCode,"用户名为："+userName+" 具体错误信息为："+e);
}
return DataXException.asDataXException(dbUtilErrorCode,"执行的SQL为: "+querySql+" 具体错误信息为："+e);
}
if (dataBaseType.equals(DataBaseType.Oracle)){
DBUtilErrorCode dbUtilErrorCode = oracleQueryErrorAna(e.getMessage());
if (dbUtilErrorCode == DBUtilErrorCode.ORACLE_QUERY_TABLE_NAME_ERROR && table != null){
return DataXException.asDataXException(dbUtilErrorCode,"表名为："+table+" 执行的SQL为:"+querySql+" 具体错误信息为："+e);
}
if (dbUtilErrorCode == DBUtilErrorCode.ORACLE_QUERY_SELECT_PRI_ERROR){
return DataXException.asDataXException(dbUtilErrorCode,"用户名为："+userName+" 具体错误信息为："+e);
}
return DataXException.asDataXException(dbUtilErrorCode,"执行的SQL为: "+querySql+" 具体错误信息为："+e);
}
return DataXException.asDataXException(DBUtilErrorCode.SQL_EXECUTE_FAIL, "执行的SQL为: "+querySql+" 具体错误信息为："+e);
}
public static DBUtilErrorCode mySqlQueryErrorAna(String e){
if (e.contains(Constant.MYSQL_TABLE_NAME_ERR1) && e.contains(Constant.MYSQL_TABLE_NAME_ERR2)){
return DBUtilErrorCode.MYSQL_QUERY_TABLE_NAME_ERROR;
}else if (e.contains(Constant.MYSQL_SELECT_PRI)){
return DBUtilErrorCode.MYSQL_QUERY_SELECT_PRI_ERROR;
}else if (e.contains(Constant.MYSQL_COLUMN1) && e.contains(Constant.MYSQL_COLUMN2)){
return DBUtilErrorCode.MYSQL_QUERY_COLUMN_ERROR;
}else if (e.contains(Constant.MYSQL_WHERE)){
return DBUtilErrorCode.MYSQL_QUERY_SQL_ERROR;
}
return DBUtilErrorCode.READ_RECORD_FAIL;
}
public static DBUtilErrorCode oracleQueryErrorAna(String e){
if (e.contains(Constant.ORACLE_TABLE_NAME)){
return DBUtilErrorCode.ORACLE_QUERY_TABLE_NAME_ERROR;
}else if (e.contains(Constant.ORACLE_SQL)){
return DBUtilErrorCode.ORACLE_QUERY_SQL_ERROR;
}else if (e.contains(Constant.ORACLE_SELECT_PRI)){
return DBUtilErrorCode.ORACLE_QUERY_SELECT_PRI_ERROR;
}
return DBUtilErrorCode.READ_RECORD_FAIL;
}
public static DataXException asSqlParserException(DataBaseType dataBaseType, Exception e,String querySql){
if (dataBaseType.equals(DataBaseType.MySql)){
throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_QUERY_SQL_PARSER_ERROR, "执行的SQL为:"+querySql+" 具体错误信息为：" + e);
}
if (dataBaseType.equals(DataBaseType.Oracle)){
throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_QUERY_SQL_PARSER_ERROR,"执行的SQL为:"+querySql+" 具体错误信息为：" +e);
}
throw DataXException.asDataXException(DBUtilErrorCode.READ_RECORD_FAIL,"执行的SQL为:"+querySql+" 具体错误信息为："+e);
}
public static DataXException asPreSQLParserException(DataBaseType dataBaseType, Exception e,String querySql){
if (dataBaseType.equals(DataBaseType.MySql)){
throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_PRE_SQL_ERROR, "执行的SQL为:"+querySql+" 具体错误信息为：" + e);
}
if (dataBaseType.equals(DataBaseType.Oracle)){
throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_PRE_SQL_ERROR,"执行的SQL为:"+querySql+" 具体错误信息为：" +e);
}
throw DataXException.asDataXException(DBUtilErrorCode.READ_RECORD_FAIL,"执行的SQL为:"+querySql+" 具体错误信息为："+e);
}
public static DataXException asPostSQLParserException(DataBaseType dataBaseType, Exception e,String querySql){
if (dataBaseType.equals(DataBaseType.MySql)){
throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_POST_SQL_ERROR, "执行的SQL为:"+querySql+" 具体错误信息为：" + e);
}
if (dataBaseType.equals(DataBaseType.Oracle)){
throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_POST_SQL_ERROR,"执行的SQL为:"+querySql+" 具体错误信息为：" +e);
}
throw DataXException.asDataXException(DBUtilErrorCode.READ_RECORD_FAIL,"执行的SQL为:"+querySql+" 具体错误信息为："+e);
}
public static DataXException asInsertPriException(DataBaseType dataBaseType, String userName,String jdbcUrl){
if (dataBaseType.equals(DataBaseType.MySql)){
throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_INSERT_ERROR, "用户名为:"+userName+" jdbcURL为："+jdbcUrl);
}
if (dataBaseType.equals(DataBaseType.Oracle)){
throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_INSERT_ERROR,"用户名为:"+userName+" jdbcURL为："+jdbcUrl);
}
throw DataXException.asDataXException(DBUtilErrorCode.NO_INSERT_PRIVILEGE,"用户名为:"+userName+" jdbcURL为："+jdbcUrl);
}
public static DataXException asDeletePriException(DataBaseType dataBaseType, String userName,String jdbcUrl){
if (dataBaseType.equals(DataBaseType.MySql)){
throw DataXException.asDataXException(DBUtilErrorCode.MYSQL_DELETE_ERROR, "用户名为:"+userName+" jdbcURL为："+jdbcUrl);
}
if (dataBaseType.equals(DataBaseType.Oracle)){
throw DataXException.asDataXException(DBUtilErrorCode.ORACLE_DELETE_ERROR,"用户名为:"+userName+" jdbcURL为："+jdbcUrl);
}
throw DataXException.asDataXException(DBUtilErrorCode.NO_DELETE_PRIVILEGE,"用户名为:"+userName+" jdbcURL为："+jdbcUrl);
}
public static DataXException asSplitPKException(DataBaseType dataBaseType, Exception e,String splitSql,String splitPkID){
if (dataBaseType.equals(DataBaseType.MySql)){
return DataXException.asDataXException(DBUtilErrorCode.MYSQL_SPLIT_PK_ERROR,"配置的SplitPK为: "+splitPkID+", 执行的SQL为: "+splitSql+" 具体错误信息为："+e);
}
if (dataBaseType.equals(DataBaseType.Oracle)){
return DataXException.asDataXException(DBUtilErrorCode.ORACLE_SPLIT_PK_ERROR,"配置的SplitPK为: "+splitPkID+", 执行的SQL为: "+splitSql+" 具体错误信息为："+e);
}
return DataXException.asDataXException(DBUtilErrorCode.READ_RECORD_FAIL,splitSql+e);
}
}
StrUtil
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class StrUtil {
private final static long KB_IN_BYTES = 1024;
private final static long MB_IN_BYTES = 1024 * KB_IN_BYTES;
private final static long GB_IN_BYTES = 1024 * MB_IN_BYTES;
private final static long TB_IN_BYTES = 1024 * GB_IN_BYTES;
private final static DecimalFormat df = new DecimalFormat("0.00");
private static final Pattern VARIABLE_PATTERN = Pattern
.compile("(\\$)\\{?(\\w+)\\}?");
private static String SYSTEM_ENCODING = System.getProperty("file.encoding");
static {
if (SYSTEM_ENCODING == null) {
SYSTEM_ENCODING = "UTF-8";
}
}
private StrUtil() {
}
public static String stringify(long byteNumber) {
if (byteNumber / TB_IN_BYTES > 0) {
return df.format((double) byteNumber / (double) TB_IN_BYTES) + "TB";
} else if (byteNumber / GB_IN_BYTES > 0) {
return df.format((double) byteNumber / (double) GB_IN_BYTES) + "GB";
} else if (byteNumber / MB_IN_BYTES > 0) {
return df.format((double) byteNumber / (double) MB_IN_BYTES) + "MB";
} else if (byteNumber / KB_IN_BYTES > 0) {
return df.format((double) byteNumber / (double) KB_IN_BYTES) + "KB";
} else {
return String.valueOf(byteNumber) + "B";
}
}
public static String replaceVariable(final String param) {
Map<String, String> mapping = new HashMap<String, String>();
Matcher matcher = VARIABLE_PATTERN.matcher(param);
while (matcher.find()) {
String variable = matcher.group(2);
String value = System.getProperty(variable);
if (StringUtils.isBlank(value)) {
value = matcher.group();
}
mapping.put(matcher.group(), value);
}
String retString = param;
for (final String key : mapping.keySet()) {
retString = retString.replace(key, mapping.get(key));
}
return retString;
}
public static String compressMiddle(String s, int headLength, int tailLength) {
Validate.notNull(s, "Input string must not be null");
Validate.isTrue(headLength > 0, "Head length must be larger than 0");
Validate.isTrue(tailLength > 0, "Tail length must be larger than 0");
if(headLength + tailLength >= s.length()) {
return s;
}
return s.substring(0, headLength) + "..." + s.substring(s.length() - tailLength);
}
}
扫一扫
693
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
