datax相关记录

186 datax相关

1.Python环境

[vmuser@186 ~]$  ll /usr/bin/pyth*

/usr/bin/python -> /usr/bin/python2.6

/usr/bin/python2.6

2.Datax 目录

/home/vmuser/datax

3. datax-web目录

/home/vmuser/datax/datax-web-2.1.2

4.启动datax-web

cd /home/vmuser/datax/datax-web-2.1.2/bin

./start-all.sh

例子中配置的，datax_web和datax_web_copy这两个库都在186mysql上，可以登录上去查看数据同步情况

6.添加执行器 正常应该是自动注册上来的，还有点问题，不过试了下手动注册也能使用。已经注册上了，现在端口1999，只需要注册一次就行了，任务由这个执行器执行具体操作。也可以添加更多的执行器，实现横向扩展，这个也可以后面考虑。

7.添加这个串指定参数   
?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8
同步之后才不会出现乱码

8.  增量同步的例子

{
   "job": {
     "setting": {
       "speed": {
         "channel": 3
       },
       "errorLimit": {
         "record": 0,
         "percentage": 0.02
       }
     },
     "content": [
       {
         "reader": {
           "name": "mysqlreader",
           "parameter": {
             "username": "yRjwDFuoPKlqya9h9H2Amg==",
             "password": "",
             "column": [
               "id",
               "job_group",
               "job_id",
               "job_desc",
               "executor_address",
               "executor_handler",
               "executor_param",
               "executor_sharding_param",
               "executor_fail_retry_count",
               "trigger_time",
               "trigger_code",
               "trigger_msg",
               "handle_time",
               "handle_code",
               "handle_msg",
               "alarm_status",
               "process_id",
               "max_id"
             ],
             "splitPk": "",
             "connection": [
               {
                 "querySql": [
                   "select * from job_log where id>= ${startId} and id< ${endId}"
                 ],
                 "jdbcUrl": [
                   "jdbc:mysql://localhost:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8"
                 ]
               }
             ]
           }
         },
         "writer": {
           "name": "mysqlwriter",
           "parameter": {
             "username": "yRjwDFuoPKlqya9h9H2Amg==",
             "password": "",
             "column": [
               "id",
               "job_group",
               "job_id",
               "job_desc",
               "executor_address",
               "executor_handler",
               "executor_param",
               "executor_sharding_param",
               "executor_fail_retry_count",
               "trigger_time",
               "trigger_code",
               "trigger_msg",
               "handle_time",
               "handle_code",
               "handle_msg",
               "alarm_status",
               "process_id",
               "max_id"
             ],
             "connection": [
               {
                 "table": [
                   "job_log"
                 ],
                 "jdbcUrl": "jdbc:mysql://localhost:3306/datax_web_copy?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8"
               }
             ]
           }
         }
       }
     ]
   }
 }