BUCHERS'

Git 版本回退

Sat, 13 Mar 2021 00:00:00 +0000

参考,部分转载:https://yorkyu.cn/how-to-use-rebase-to-gracefully-fallback-code-08cdafe3d403.html

基本操作概念

Git Revert 使用命令 git revert commit_id 能产生一个与 commit_id 完全相反的提交，即在 log 中会看到一条新的提交 new_commit_id， revert 提交就是删除 commit_id 的提交。
Git Reset reset 也能使代码回到某次提交，但跟 revert 不同的是， reset 是将提交的 HEAD 指针指到某次提交，之后的提交记录会消失，就像从没有过这么一次提交
Get Rebase rebase 是“变基”的意思，这里的“基”，指[多次] commit 形成的 git workflow，使用 rebase，我们可以改变这些历史提交，修改 commit 信息，将多个 commit 进行组合。

Rebase+Revert回退

切出一个新分支 feature1 首先，切出一个新分支 feature1，使用 git log 查询一下要回退到的 commit 版本 commit_n。
feature1 分支执行命令 git rebase -i commit_n
feature1 合并 commit2 ~ commit4 到最旧的 commit1 上在合并 commit 时，我们可以选择 pick(p) 最旧的 commit1，然后在后续的 commit_xxx 前添加 squash(s) 命令，将这些 commits 都合并到最旧的 commit1 上。
```
pick 6fa5869 commit1
squash 0b84ee7 commit2
squash 986c6c8 commit3
squash 91a0dcc commit4
```

feature1 合并提交记录

# 执行rebase 生成的新记录
commit 5
# 回退代码的版本号
commit_n

feature1 合并 master 到 feature1
由于 feature1 分支落后与 master 分支，因此需要执行 git merge master 将master 分支向 feature1 分支合并。合并后 git 会发现 commit1 到 commit4 提交的内容和 feature1 分支上 commit5 的修改内容是完全相同的，会自动进行合并，内容不变，但多了一个 commit5。
```
# feature1 分支
commit 5
commit 4
commit 3
commit 2
commit 1
# 回退代码的版本号
commit_n
```
feature1 分支执行 revert 反提交
在 feature1 分支上对 commit5 进行一次 revert 反提交，就实现了把 commit1 到 commit4 的提交全部回退。即撤回步骤5中的合并。
```
git revert commit_5
# 执行rebase 生成的新记录
commit 5
# 回退代码的版本号
commit_n
```
master 分支执行合并feature1到master
master 代码回退，但保留了commit_n, 1,2,3,4,5所有记录
```
# 切回到 master
$ git checkout master
# 合并 feature1
$ git merge feature1
```

循环fibnaci

Wed, 13 Feb 2019 00:00:00 +0000

package com.test;

import java.util.Stack;

public class FibnaciStack {
    public static <T> void printStack(Stack<T[]> stack) {
        System.out.println("STACK:BEGIN");
        int STACK_SIZE = stack.size();
        System.out.println("b,s,n");
        for (int i = STACK_SIZE; i > 0; i--) {
            T[] objs = stack.get(i - 1);
            for (T obj : objs) {
                System.out.print(obj + ",");
            }
            System.out.println();
        }
        System.out.println("STACK:END");
    }

    public static void c2(int n) {
        long s = 0, rt = 0;
        int branch = 0;
        Stack<Object[]> stack = new Stack<>();
        while (true) {
            if (n < 2) {
                // this will goto last one, as nothing to operate
                // pop next things to operate, define $rt as last result in {0,1}
                // we have the result, there is no need for current stack.
                rt = n;
                System.out.println("Print $$n<2, n=" + n);
            } else {
                switch (branch) {
                    // Push F(n), F(n-2), F(n-2-2)
                    // TODO STEP1 压栈操作
                    case 0: {
                        do {
                            stack.push(new Object[]{branch, 0l, n});
                            n -= 2;
                        } while (n >= 2);
                        System.out.print("branch=0,");
                        printStack(stack);
                        System.out.println("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
                        continue;
                    }
                    // F(n+1) = F(n) + F(n-1)
                    // Push F(n) and handle F(n-1)
                    // TODO STEP2 保留 $rt 结果，F(n) 分支压栈; F(n-1) 到 $branch0 再压栈
                    // $branch=1 的情况就是有 $rt 中间结果
                    case 1: {
                        // push the F(n):
                        // 1. $branch will be 2 when (n--) <=2 in next step;(Refer TODO $2)
                        // 2. Otherwise $branch will remain 0 for stack push(Refer TODO $1)
                        // 最终结果还是保留在了 $s=$rt，中间结果
                        stack.push(new Object[]{branch, rt, n});
                        // process F(n-1):
                        // Goto $BRANCH=0 if (n--)>=2 ;
                        // Otherwise goto n<2 and then go to TODO $$Popup;
                        n--;
                        // goto branch0,将 n-- 结果压栈
                        // TODO $1
                        branch = 0;
                        System.out.print("branch=1,");
                        printStack(stack);
                        System.out.println("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
                        continue;
                    }
                    /** TODO STEP3 更新当前 $rt:
                     *
                     *  1. n >=2 时, $rt 下次直接被保留 到 $branch1 压栈
                     *  2. n < 2 时，$rt 会被跟新 $rn = n
                     */

                    case 2: {
                        // 1. calc rs til now in one F(n),calc from $BRANCH=1, $BRANCH++
                        // we have the result, there is no need for current stack.
                        // 2. Stack will be popup, if the popped $branch==1, we
                        // will be here again
                        System.out.println("We have the previous Result!!");
                        rt += s;
                        break;
                    }
                    default:
                        break;
                }
            }
            if (stack.isEmpty()) {
                System.out.println("Result is :" + rt);
                return;
            }
            // TODO $$Popup
            /**
             * TODO STEP4 弹出操作栈
             * 1. 弹出 branch0 ->1 下次操作保留 TODO $rt;
             * 2. 弹出 branch1 ->2 下次操作更新 TODO $rt+=s;
             */

            Object[] t = stack.pop();
            branch = (Integer) t[0];
            s = (Long) t[1];
            n = (Integer) t[2];
            System.out.println("POPUP Branch=" + branch + " s=" + s + " n=" + n + " rt=" + rt);
            printStack(stack);
            System.out.println("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
            // TODO $2
            branch++;

        }
    }

    public static void main(String[] args) {
        c2(6);
    }

}

Cas exchanger

Fri, 25 Jan 2019 00:00:00 +0000

package com.test.distrib;

import java.io.PrintStream;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;

public class FileDistributionStreamExchangeTask implements Runnable {

    private static final String counterGrp = "edit-counters";
    private static final String storeName1 = "wikipedia-stats1";
    private static final String storeName2 = "wikipedia-stats2";
    private static final String storeKey1 = "Hello1";
    private static final String storeKey2 = "Hello2";
    private static final PrintStream OUTPUT_STREAM = System.out;
    private static final String counterName1 = "repeat-edits1";
    private static final String counterName2 = "repeat-edits2";

    private DistributionStreamWorker worker1;
    private DistributionStreamWorker worker2;

    private static final long limits = 2000;

    private volatile DistributionStreamWorker cworker;
    private volatile AtomicReferenceFieldUpdater<FileDistributionStreamExchangeTask, DistributionStreamWorker> storeCAS =
            AtomicReferenceFieldUpdater.newUpdater(FileDistributionStreamExchangeTask.class,
                    DistributionStreamWorker.class,
                    "cworker");


    public FileDistributionStreamExchangeTask() {
        init();
    }

    public void init() {
        Map<String, Map<String, List<String>>> taskContext = new HashMap<>();
        taskContext.put(storeName1, new HashMap<>());
        taskContext.put(storeName2, new HashMap<>());
        this.worker1 = new DistributionStreamWorker(storeKey1, new AtomicLong(), taskContext.get(storeName1));
        this.worker2 = new DistributionStreamWorker(storeKey2, new AtomicLong(), taskContext.get(storeName2));
        // cworker should be null, init once
        storeCAS.getAndSet(this, worker1);
        Timer t = new Timer();
        t.schedule(new TimerTask() {
            @Override
            public void run() {
                window();
            }
        }, 0, 100);

    }

    public void process(IncomingMessageEnvelope envelope) {
        DistributionStreamWorker sWorker = cworker;
        while (sWorker.counter.get() >= limits) {// TODO 1
            // TODO 2
            if (sWorker == worker1) {
                if (!storeCAS.compareAndSet(this, worker1, worker2)) {
                    sWorker = cworker;
                    System.out.println("Swap 1 - 2");
                    continue;
                }
            } else {
                if (!storeCAS.compareAndSet(this, worker2, worker1)) {
                    sWorker = cworker;
                    System.out.println("Swap 2 - 1");
                    continue;
                }
            }
            sWorker.purge(this.limits);
        }
		// TODO 5
        cworker.handleMsg(envelope);
    }


    public void window() {
        // Send the remains
        System.out.println("window flush==============================");
        cworker.purge(-1);
    }

    @Override
    public void run() {

        while (true) {
            IncomingMessageEnvelope msg = new IncomingMessageEnvelope();
            Map<String, Object> map = new HashMap<>();
            map.put("orderDetails", "Generator " + Thread.currentThread().getId());
            msg.setMessage(map);
            map.put("" + System.currentTimeMillis(), "Thread " + Thread.currentThread().getId() + ": some msg");
            process(msg);
        }
    }

    private static class IncomingMessageEnvelope {
        private Map<String, Object> message;


        public Map<String, Object> getMessage() {
            return message;
        }

        public void setMessage(Map<String, Object> message) {
            this.message = message;
        }
    }


    private static class DistributionStreamWorker {
        volatile AtomicLong counter;
        volatile String batchInfo = null;
        Map<String, List<String>> store;
        String storeKey;

        private DistributionStreamWorker(String storeKey, AtomicLong counter, Map<String, List<String>> store) {
            this.counter = counter;
            this.store = store;
            this.storeKey = storeKey;
        }

        public synchronized void purge(long limit) {
            // TODO 3
            if (limit == -1 || counter.get() >= limit) {
                flush();
            }
        }

        private void flush() {
            List<String> stack = this.store.get(storeKey);
            if (stack == null) {
                return;
            }
            OUTPUT_STREAM.println("Start to flush" + this.store.get(storeKey).size());
            this.store.remove(storeKey);
            counter.set(0l);
            batchInfo = "";
        }

        public synchronized void handleMsg(IncomingMessageEnvelope envelope) {
			// TODO 4
            if (counter.get() >= limits) {
                flush();
            }
            System.out.println("Clean" + Thread.currentThread().getId());
            Map<String, Object> edit = envelope.getMessage();
            List<String> arr = this.store.get(storeKey);
            if (arr == null) {
                arr = new ArrayList<>();
                this.store.put(storeKey, arr);
            }
            String msg = edit.get("orderDetails").toString();
            long count = this.counter.get();
            if (count > limits) {
                System.err.println(count + "Out of limists!!!!");
                System.exit(0);
            }
            System.out.println("Total :" + count + "Current Consumer ===" + Thread.currentThread().getId() +
                    "+++++++++++" + msg);
            arr.add(msg);
            // Update Batch here if possible
            batchInfo += 1;
            this.counter.getAndIncrement();
        }

    }

    public static void main(String[] args) throws InterruptedException {
        final FileDistributionStreamExchangeTask t = new FileDistributionStreamExchangeTask();
        Thread[] tgroup = new Thread[2];
        for (int i = 0; i < tgroup.length; i++) {
            tgroup[i] = new Thread(t);
            tgroup[i].join();
        }
        for (int i = 0; i < tgroup.length; i++) {
            tgroup[i] = new Thread(t);
            tgroup[i].start();
        }

    }
}

Kafka cluaster partitions replicas group

Sun, 20 Jan 2019 00:00:00 +0000

Partition & Replicas

Kafka 集群默认自动分配解析

下面以一个Kafka集群中4个Broker举例，创建1个topic包含4个Partition，2 Replication；数据Producer流动如图所示：

Broker1	Broker2	Broker3	Broker4	BrokerX	BrokerX
P0	P1	P2	P3	N/A	N/A
P3	P0	P1	P2	N/A	N/A

当集群中新增2节点，Partition增加到6个时分布情况如下：

Broker1	Broker2	Broker3	Broker4	Broker5	Broker6
P0	P1	P2	P3	P4	P5
P3	P0	P1	P2		P4
P5

副本分配逻辑规则如下：

在Kafka集群中，每个Broker都有均等分配Partition的Leader机会。
上述图Broker Partition中，箭头指向为副本，以Partition-0为例:broker1中parition-0为Leader，Broker2中Partition-0为副本。
上述图种每个Broker(按照BrokerId有序)依次分配主Partition,下一个Broker为副本，如此循环迭代分配，多副本都遵循此规则。

副本分配算法如下：

将所有N Broker和待分配的i个Partition排序.
将第i个Partition分配到第(i mod n)个Broker上.
将第i个Partition的第j个副本分配到第((i + j) mod n)个Broker上.

Group on Consumers

Kafka Partition & Group

原理图
原理描述

一个topic 可以配置几个partition，produce发送的消息分发到不同的partition中，consumer接受数据的时候是按照group来接受，kafka确保每个partition只能同一个group中的同一个consumer消费，如果想要重复消费，那么需要其他的组来消费。Zookeerper中保存这每个topic下的每个partition在每个group中消费的offset  
新版kafka把这个offsert保存到了一个__consumer_offsert的topic下  
这个__consumer_offsert 有50个分区，通过将group的id哈希值%50的值来确定要保存到那一个分区.  这样也是为了考虑到zookeeper不擅长大量读写的原因。  
所以，如果要一个group用几个consumer来同时读取的话，需要多线程来读取，一个线程相当于一个consumer实例。当consumer的数量大于分区的数量的时候，有的consumer线程会读取不到数据。   
假设一个topic test 被groupA消费了，现在启动另外一个新的groupB来消费test，默认test-groupB的offset不是0，而是没有新建立，除非当test有数据的时候，groupB会收到该数据，该条数据也是第一条数据，groupB的offset也是刚初始化的ofsert, 除非用显式的用–from-beginnging 来获取从0开始数据   

查看topic-group的offsert

位置：zookeeper 
路径：[zk: localhost:2181(CONNECTED) 3] ls /brokers/topics/__consumer_offsets/partitions 
在zookeeper的topic中有一个特殊的topic __consumer_offserts 
计算方法：（放入哪个partitions）

int hashCode = Math.abs("ttt".hashCode());
int partition = hashCode % 50;
先计算group的hashCode，再除以分区数(50),可以得到partition的值 
使用命令查看： kafka-simple-consumer-shell.sh --topic __consumer_offsets --partition 11 --broker-list localhost:9092,localhost:9093,localhost:9094 --formatter "kafka.coordinator.GroupMetadataManager\$OffsetsMessageFormatter"

参数
auto.offset.reset:默认值为largest，代表最新的消息，smallest代表从最早的消息开始读取，当consumer刚开始创建的时候没有offset这种情况，如果设置了largest，则为当收到最新的一条消息的时候开始记录offsert,若设置为smalert，那么会从头开始读partition
Consumer Group 使用Consumer high level API时，同一Topic的一条消息只能被同一个Consumer Group内的一个Consumer消费，但多个Consumer Group可同时消费这一消息。
这是Kafka用来实现一个Topic消息的广播（发给所有的Consumer）和单播（发给某一个Consumer）的手段。一个Topic可以对应多个Consumer Group。如果需要实现广播，只要每个Consumer有一个独立的Group就可以了。要实现单播只要所有的Consumer在同一个Group里。用Consumer Group还可以将Consumer进行自由的分组而不需要多次发送消息到不同的Topic。
实际上，Kafka的设计理念之一就是同时提供离线处理和实时处理。根据这一特性，可以使用Storm这种实时流处理系统对消息进行实时在线处理，同时使用Hadoop这种批处理系统进行离线处理，还可以同时将数据实时备份到另一个数据中心，只需要保证这三个操作所使用的Consumer属于不同的Consumer Group即可。
下面这个例子更清晰地展示了Kafka Consumer Group的特性。首先创建一个Topic (名为topic1，包含3个Partition)，然后创建一个属于group1的Consumer实例，并创建三个属于group2的Consumer实例，最后通过Producer向topic1发送key分别为1，2，3的消息。结果发现属于group1的Consumer收到了所有的这三条消息，同时group2中的3个Consumer分别收到了key为1，2，3的消息。

List node 节点大树算法

Sun, 02 Dec 2018 00:00:00 +0000

只适合差一位和相同位数两数相加；若需要实现相差多位数相加，需要另外探深度

package ext.lc;

import java.util.HashSet;
import java.util.Set;

public class Solution {

    private static Set<ListNode> l = new HashSet<>();
    public static int[] vlist = new int[10];

    public static ListNode addTwoNumbers(ListNode n1, ListNode n2) {
        return addNext(n1, n2, null, null, 0);
    }

    public static ListNode addNext(ListNode n1, ListNode n2, ListNode p1,
                                   ListNode p2, int level) {
        if (n1 == null && n2 == null) {
            return null;
        } else if (n1 != null && n2 != null) {
            System.out.println("N1: " + n1.val + " N2:" + n2.val);
            if (n1.next != null || n2.next != null) {
                if (n1.next != null ^ n2.next != null) {
                    // At least one n.next is not null, recurse to find next
                    // 1,2,3 (next 3 is not null)
                    // 1,2,(2)(next is null, align to next
                    ListNode temp = addNext(n1.next == null ? n1 : n1.next,
                            n2.next == null ? n2 : n2.next, n1, n2, level + 1);
                    int val = temp.val;
                    temp.val = val % 10;
                    l.add(temp);
                    // 1,2,3
                    //   1,2(1 is p1/p2, whose next is null)
                    temp = new ListNode((n1.next == null ? p1.val : n1.val) + (n2.next == null ? p2.val : n2.val) + val / 10);
                    val = temp.val;
                    temp.val = val % 10;
                    // Update p for val > 9
                    if (p1 != null) {
                        p1.val += val / 10;
                    } else if (p2 != null) {
                        p2.val += val / 10;
                    }
                    // FIXME POS_1
                    if (n1.next == null) {
                        n1.val = p1.val;
                        n1.next = p1.next;
                    } else if (n2.next == null) {
                        n2.val = p2.val;
                        n2.next = p2.next;
                    }
                    vlist[level] = temp.val;
                    l.add(temp);
                    return temp;
                } else {
                    // all next are not null
                    ListNode temp = addNext(n1.next, n2.next, n1, n2,
                            level + 1);
                    l.add(temp);
                    // All next are not null, n is loop equals. n.next == n
                    // .next.net
                    // If so, n renew with p next for further check.
                    // Initialed by FIXME POS_1
                    renewNode(n1, p1);
                    renewNode(n2, p2);
                    int val = n1.val + n2.val;
                    temp = new ListNode(val % 10);
                    if (p1 != null) {
                        p1.val += val / 10;
                    } else if (p2 != null) {
                        p2.val += val / 10;
                    }
                    vlist[level] = temp.val;
                    l.add(temp);
                    return temp;
                }
            }
            // all next null, return n1 + n2
            int val = n1.val + n2.val;
            ListNode temp = new ListNode(val % 10);
            if (p1 != null) {
                p1.val += val / 10;
            } else if (p2 != null) {
                p2.val += val / 10;
            }
            vlist[level] = temp.val;
            l.add(temp);
            return temp;
        } else {
            // at least 1 null
            ListNode tNode = addNext(n1 == null ? null : n1.next, n2 == null ?
                    null : n2.next, n1, n2, level + 1);
            tNode = tNode == null ? (n1 == null ? n2 : n1) : tNode;
            vlist[level] = tNode.val;
            l.add(tNode);
            return tNode;
        }
    }

    /**
     * Check if n is loop equals n.next == n.next.next;
     * Renew n to p if p is not null.
     *
     * @param n
     * @param p
     */
    private static void renewNode(ListNode n, ListNode p) {
        if (n.next != null && n.next == n.next.next) {
            if (p == null) {
                n.val = 0;
            } else {
                n.val = p.val;
                n.next = p.next;
            }
        }
    }

    public static void main(String[] args) {
        // Init list1 7247
        ListNode n1 = new ListNode(7);
        ListNode n2 = new ListNode(2);
        ListNode n3 = new ListNode(4);
        ListNode n4 = new ListNode(7);
        n1.next = n2;
        n2.next = n3;
        n3.next = n4;
        // 7247
        //  954
        // 8201
        ListNode n11 = new ListNode(9);
        ListNode n12 = new ListNode(5);
        ListNode n13 = new ListNode(4);
        n11.next = n12;
        n12.next = n13;
        for (int i = 0; i < 10; i++) {
            System.out.print(vlist[i]);
        }
        System.out.println();
    }
}

Winscp_scripts

Sat, 03 Nov 2018 00:00:00 +0000

scp.bat

SET rootPath="C:\Program Files (x86)\WinSCP\"
SET folderDate=%date:~10,4%-%date:~4,2%-%date:~7,2%
mkdir e:\%folderDate%
SET localFilePath=e:\%folderDate%
%rootPath%\WinSCP.exe /console /script="C:\Users\yuanhui\Desktop\GetValidationData.txt"

GetValidationData.txt

option batch continue
option confirm off
option transfer binary
open sftp://user:psw@localhost:22
cd /data/streaming/%folderDate%
lcd %localFilePath%
get *
close
exit

Password expire

Sat, 03 Nov 2018 00:00:00 +0000

TODAY=`date "+%b %d, %Y"`
cat vmlist.txt|while read line;do
    EXPIRE_DATE=`ssh -n $line "chage -l wmuser|grep 'expires';exit"|tail|awk '{print $4,$5,$6}'`;
    echo $EXPIRE_DATE;
    echo $TODAY;
    t1=`date -d "$EXPIRE_DATE" +%s`;
    t2=`date -d "$TODAY" +%s`;
    if [ "$t1" -gt "$t2" ]; then
        echo "$EXPIRE_DATE > $TODAY"
    else
        echo "$TODAY > $EXPIRE_DATE"
    fi;
done;

Grafana环境搭建

Sat, 08 Sep 2018 00:00:00 +0000

Install sqlite3

sudo wget https://www.sqlite.org/2018/sqlite-tools-linux-x86-3240000.zip

Install mysql

wget http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm
sudo rpm -ivh mysql-community-release-el7-5.noarch.rpm
sudo yum install mysql-community-server

service mysqld start/stop

mysql -u root
CREATE DATABASE grafana_db
insert into mysql.user(Host,User,Password) values("localhost","grafanaReader",password("password"));
flush privileges;
grant all privileges on granfana_db.* to grafanaReader@localhost identified by 'password';
flush privileges;

mysql -u grafanaReader -p

Install grafana

touch bin/grafana.db
sudo bin/grafana-server -config="conf/defaults.ini"

Flumeagentsource搭建

Fri, 07 Sep 2018 00:00:00 +0000

keytool

生成key

keytool -genkeypair -alias avro -keyalg RSA -keystore mykeystore/keystore.jks -dname "CN=localhost, OU=localhost,O=localhost, L=SH, ST=SH, C=CN" -keypass changeit -storepass changeit

查阅

keytool -list -v -alias avro -keystore mykeystore/keystore.jks -storepass changeit

导出

keytool -export -alias avro -keystore mykeystore/keystore.jks -rfc -file mykeystore/mycert.cer

生成 truststore

keytool -import -alias avro -file mykeystore/mycert.cer  -keystore truststore.jks

Avro Source Agent

Command

sudo bin/flume-ng agent -n agent -c conf -f conf/avro-src.conf -Dflume.root.logger=INFO,console

Config

agent.sources = src-avro
agent.channels = memoryChannel
agent.sinks = loggerSink
# For each one of the sources, the type is defined
agent.sources.src-avro.type =avro
# The channel can be defined as follows.
agent.sources.src-avro.channels = memoryChannel
agent.sources.src-avro.bind=127.0.0.1
agent.sources.src-avro.port=1000
agent.sources.src-avro.ssl=true
agent.sources.src-avro.keystore=/home/yuanhui0628/temp/mykeystore/keystore.jks
agent.sources.src-avro.keystore-password=changeit
agent.sources.src-avro.keystore-type=JKS
# Each sink's type must be defined
agent.sinks.loggerSink.type = logger
#Specify the channel the sink should use
agent.sinks.loggerSink.channel = memoryChannel
# Each channel's type is defined.
agent.channels.memoryChannel.type = memory
# Other config values specific to each type of channel(sink or source)
# can be defined as well
# In this case, it specifies the capacity of the memory channel
agent.channels.memoryChannel.capacity = 100 

Avro Sink Agent

Command

bin/flume-ng agent -n agent -c conf -f conf/avro-agent.conf -Dflume.root.logger=INFO,console

Config

agent.sources = netSrc
agent.channels = memoryChannel
agent.sinks = avro-sink
# For each one of the sources, the type is defined
agent.sources.netSrc.type = netcat
agent.sources.netSrc.bind= 127.0.0.1
agent.sources.netSrc.port= 9999
# The channel can be defined as follows.
agent.sources.netSrc.channels = memoryChannel
# Each sink's type must be defined
agent.sinks.avro-sink.type = avro
#Specify the channel the sink should use
agent.sinks.avro-sink.channel = memoryChannel
agent.sinks.avro-sink.hostname= 127.0.0.1
agent.sinks.avro-sink.port = 1000

agent.sinks.avro-sink.ssl=true
agent.sinks.avro-sink.trust-all-certs=true
agent.sinks.avro-sink.truststrore=/home/yuanhui0628/temp/mykeystore/truststore.jks
agent.sinks.avro-sink.truststrore-password=changeit
agent.sinks.avro-sink.truststrore-type=JKS
# Each channel's type is defined.
agent.channels.memoryChannel.type = memory
# Other config values specific to each type of channel(sink or source)
# can be defined as well
# In this case, it specifies the capacity of the memory channel
agent.channels.memoryChannel.capacity = 100 

标志位配置设计

Tue, 10 Jul 2018 00:00:00 +0000

枚举包装接口

public interface FilterEnums {

    Enumeration<FlagStatsCode> elements();
    int length();

}

实现

public class StatCodeEnumerator implements FilterEnums {

    private FlagStatsCode[] codes;


    public StatCodeEnumerator(FlagStatsCode[] codes) {
        assert (codes != null);
        this.codes = codes;

    }

    @Override
    public Enumeration<FlagStatsCode> elements() {
        return new Enumeration<FlagStatsCode>() {
            private int count = 0;

            @Override
            public boolean hasMoreElements() {
                return count < StatCodeEnumerator.this.codes.length;
            }

            @Override
            public FlagStatsCode nextElement() {
                return StatCodeEnumerator.this.codes[count++];
            }
        };
    }

    @Override
    public int length() {
        return codes.length;
    }
}

状态Entries

public class FlagStatsCode {

    private int code;
    private String stats;

    public FlagStatsCode(String stats, int code) {
        this.code = code;
        this.stats = stats;
    }

    public int getCode() {
        return code;
    }

    public void setCode(int code) {
        this.code = code;
    }

    public String getStats() {
        return stats;
    }

    public void setStats(String stats) {
        this.stats = stats;
    }
}

聚合计算器

public class FilterAggregator {
    final int F_FLAG = 0xffffffff;
    final int H_FLAG = 0xffff0000;
    final int L_FLAG = 0x0000ffff;

    private int allowedStatus = 0;

    StatCodeEnumerator userFilter = null;

    public FilterAggregator(StatCodeEnumerator userFilter) {
        assert (userFilter != null);
        this.userFilter = userFilter;
        validate();
    }

    public int aggStatus(int mix_signal) {
        if (allowedStatus != (allowedStatus | mix_signal)) {
            throw new RuntimeException("Can't match signal");
        }
        return allowedStatus & mix_signal;
    }

    public List<FlagStatsCode> aggStatusDef(int mix_signal) {
        List<FlagStatsCode> filters = new ArrayList<>();
        if (allowedStatus != (allowedStatus | mix_signal)) {
            throw new RuntimeException("Can't match signal");
        }
        int op = allowedStatus & mix_signal;
        Enumeration<FlagStatsCode> codes = userFilter.elements();
        while (codes.hasMoreElements()) {
            FlagStatsCode def = codes.nextElement();
            int code = def.getCode();
            if ((op & code) == code) {
                filters.add(new FlagStatsCode(def.getStats(), code));
            }
        }
        return filters;
    }

    public void validate() {
        int rec_flags = 0;
        Enumeration<FlagStatsCode> codes = userFilter.elements();
        while (codes.hasMoreElements()) {
            int f = codes.nextElement().getCode();
            if ((f & H_FLAG) == 0 && (f & L_FLAG) != 0 && (f & rec_flags) == 0) {
                int check_point = 1;
                int eql_count = 0;
                for (int i = 0; i < 32; i++) {
                    if ((f & check_point) != 0) {
                        eql_count++;
                    }
                    check_point <<= 1;
                }
                if (eql_count != 1) {
                    System.out.println("eql_count " + eql_count + ", flag " + f + " is not valid!");
                    continue;
                }
                rec_flags |= f;
                continue;
            }
            System.out.println(f + " is not valid!");
        }
        allowedStatus = rec_flags;
    }


    public static void main(String[] args) {
        FlagStatsCode d1 = new FlagStatsCode("CASH", 1);
        FlagStatsCode d2 = new FlagStatsCode("WEIXIN", 2);
        FlagStatsCode d3 = new FlagStatsCode("ALI", 4);
        FlagStatsCode d4 = new FlagStatsCode("CARD", 8);
        FlagStatsCode d48 = new FlagStatsCode("48_WRONG", 48);
        FilterAggregator s = null;
        s = new FilterAggregator(new StatCodeEnumerator(new FlagStatsCode[]{d1, d2, d3, d4, d48}));
        s = new FilterAggregator(new StatCodeEnumerator(new FlagStatsCode[]{d1, d2, d3}));
        s = new FilterAggregator(new StatCodeEnumerator(new FlagStatsCode[]{d1, d3}));
        System.out.println(s.aggStatus(5));
        s = new FilterAggregator(new StatCodeEnumerator(new FlagStatsCode[]{d1, d2, d3, d4}));
        for (FlagStatsCode d : s.aggStatusDef(5)) {
            System.out.println(d.getStats() + " - " + d.getCode());
        }
    }
}

Hadoop ha +zookeeper +yarn

Sun, 29 Apr 2018 00:00:00 +0000

Hadoop 配置

hadoop配置主要涉及hdfs-site.xml，core-site.xml，mapred-site.xml，yarn-site.xml四个文件。以下详细介绍每个文件的配置。

core-site.xml

<configuration>  
<property>  
      <name>fs.defaultFS</name>  
      <value>hdfs://cluster1</value>  
      <description>HDFS namenode的逻辑名称，也就是namenode HA,此值要对应hdfs-site.xml里的dfs.nameservices</description>  
</property>  
<property>  
    <name>hadoop.tmp.dir</name>  
    <value>/home/hadoop/bigdata/tmp</value>  
    <description>hdfs中namenode和datanode的数据默认放置路径，也可以在hdfs-site.xml中分别指定</description>  
</property>  
<property>  
        <name>ha.zookeeper.quorum</name>  
        <value>m7-psdc-kvm01:2181,m7-psdc-kvm02:2181,m7-psdc-kvm03:2181</value>  
        <description>zookeeper集群的地址和端口，zookeeper集群的节点数必须为奇数</description>  
</property>  
</configuration>  

hdfs-site.xml的配置（重点配置）

<configuration>  
    <property>  
        <name>dfs.name.dir</name>  
        <value>/home/hadoop/bigdata/nn</value>  
        <description>namenode的数据放置目录</description>  
    </property>  
    <property>  
        <name>dfs.data.dir</name>  
        <value>/home/hadoop/bigdata/dn</value>  
        <description>datanode的数据放置目录</description>  
    </property>  
    <property>  
        <name>dfs.replication</name>  
        <value>2</value>  
        <description>数据块的备份数，默认是3</description>  
    </property>  
    <property>  
        <name>dfs.nameservices</name>  
        <value>cluster1</value>  
        <description>HDFS namenode的逻辑名称，也就是namenode HA</description>  
    </property>  
    <property>  
        <name>dfs.ha.namenodes.cluster1</name>  
        <value>ns1,ns2</value>  
        <description>nameservices对应的namenode逻辑名</description>  
    </property>  
    <property>  
        <name>dfs.namenode.rpc-address.cluster1.ns1</name>  
        <value>m7-psdc-kvm01:8020</value>  
        <description>指定namenode(ns1)的rpc地址和端口</description>  
    </property>  
    <property>  
        <name>dfs.namenode.http-address.cluster1.ns1</name>  
        <value>m7-psdc-kvm01:50070</value>  
        <description>指定namenode(ns1)的web地址和端口</description>  
    </property>  
    <property>  
        <name>dfs.namenode.rpc-address.cluster1.ns2</name>  
        <value>m7-psdc-kvm02:9000</value>  
        <description>指定namenode(ns2)的rpc地址和端口</description>  
    </property>  
    <property>  
        <name>dfs.namenode.http-address.cluster1.ns2</name>  
        <value>m7-psdc-kvm02:50070</value>  
        <description>指定namenode(ns2)的web地址和端口</description>  
    </property>  
    <property>  
        <name>dfs.namenode.shared.edits.dir</name>  
        <value>qjournal://m7-psdc-kvm01:8485;m7-psdc-kvm02:8485;m7-psdc-kvm03:8485;m7-psdc-kvm04:8485/cluster1 </value>  
        <description>这是NameNode读写JNs组的uri，active NN 将 edit log 写入这些JournalNode，而 standby NameNode 读取这些 edit log，并作用在内存中的目录树中</description>  
    </property>  
    <property>  
        <name>dfs.journalnode.edits.dir</name>  
        <value>/home/hadoop/bigdata/journal</value>  
        <description>journalNode 所在节点上的一个目录，用于存放 editlog 和其他状态信息。</description>  
    </property>  
    <property>  
           <name>dfs.ha.automatic-failover.enabled</name>  
           <value>true</value>  
           <description>启动自动failover。自动failover依赖于zookeeper集群和ZKFailoverController（ZKFC），后者是一个zookeeper客户端，用来监控NN的状态信息。每个运行NN的节点必须要运行一个zkfc</description>  
    </property>  
    <property>  
        <name>dfs.client.failover.proxy.provider.cluster1</name>  
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>  
        <description>配置HDFS客户端连接到Active NameNode的一个java类</description>  
    </property>  
    <property>  
        <name>dfs.ha.fencing.methods</name>  
        <value>sshfence</value>  
        <description>解决HA集群脑裂问题（即出现两个 master 同时对外提供服务，导致系统处于不一致状态）。在 HDFS HA中，JournalNode 只允许一个 NameNode 写数据，不会出现两个 active NameNode 的问题,但是，当主备切换时，之前的 active NameNode 可能仍在处理客户端的 RPC 请求，为此，需要增加隔离机制（fencing）将之前的 active NameNode 杀死。常用的fence方法是sshfence，要指定ssh通讯使用的密钥dfs.ha.fencing.ssh.private-key-files和连接超时时间</description>  
    </property>  
    <property>  
        <name>dfs.ha.fencing.ssh.private-key-files</name>  
        <value>/home/hadoop/.ssh/id_rsa</value>  
        <description>ssh通讯使用的密钥</description>  
    </property>  
    <property>  
        <name>dfs.ha.fencing.ssh.connect-timeout</name>  
        <value>30000</value>  
        <description>连接超时时间</description>  
    </property>  
</configuration>  

mapred-site.xml的配置

<configuration>  
<property>  
        <name>mapreduce.framework.name</name>  
        <value>yarn</value>  
        <description>指定运行mapreduce的环境是yarn，与hadoop1截然不同的地方</description>  
</property>  
<property>  
        <name>mapreduce.jobhistory.address</name>  
        <value>m7-psdc-kvm01:10020</value>  
         <description>MR JobHistory Server管理的日志的存放位置</description>  
</property>  
<property>  
        <name>mapreduce.jobhistory.webapp.address</name>  
        <value>m7-psdc-kvm01:19888</value>  
        <description>查看历史服务器已经运行完的Mapreduce作业记录的web地址，需要启动该服务才行</description>  
</property>  
<property>  
   <name>mapreduce.jobhistory.done-dir</name>  
   <value>/home/hadoop/bigdata/done</value>  
   <description>MR JobHistory Server管理的日志的存放位置,默认:/mr-history/done</description>  
</property>  
<property>  
   <name>mapreduce.jobhistory.intermediate-done-dir</name>  
   <value>hdfs://mycluster-pha/mapred/tmp</value>  
   <description>MapReduce作业产生的日志存放位置，默认值:/mr-history/tmp</description>  
</property>  
</configuration>  

yarn-site.xml

<configuration>  
  
  
<!-- Site specific YARN configuration properties -->  
    <property>  
                <name>yarn.resourcemanager.ha.enabled</name>  
                <value>true</value>  
        </property>  
    <property>  
                <name>yarn.resourcemanager.cluster-id</name>  
                <value>yarn-cluster1</value>  
        </property>  
<!-- 指定RM的名字 -->  
        <property>  
                <name>yarn.resourcemanager.ha.rm-ids</name>  
                <value>rm1,rm2</value>  
        </property>  
<!-- 分别指定RM的地址 -->  
        <property>  
                <name>yarn.resourcemanager.hostname.rm1</name>  
                <value>m7-psdc-kvm03</value>  
        </property>  
        <property>  
                <name>yarn.resourcemanager.hostname.rm2</name>  
                <value>m7-psdc-kvm02</value>  
        </property>  
    <property>  
                <name>yarn.resourcemanager.recovery.enabled</name>  
                <value>true</value>  
        </property>  
  
  
        <property>  
                <name>yarn.resourcemanager.store.class</name>  
                <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>  
        </property>  
        <!-- 指定zk集群地址 -->  
        <property>  
                <name>yarn.resourcemanager.zk-address</name>  
                <value>m7-psdc-kvm01:2181,m7-psdc-kvm02:2181,m7-psdc-kvm03:2181</value>  
        </property>  
    <property>  
        <name>yarn.resourcemanager.ha.automatic-failover.embedded</name>  
        <value>true</value>  
    </property>  
    <property>  
        <name>yarn.resourcemanager.admin.address.rm1</name>  
        <value>m7-psdc-kvm03:8033</value>  
    </property>  
    <property>  
        <name>yarn.resourcemanager.admin.address.rm2</name>  
        <value>m7-psdc-kvm02:8033</value>  
    </property>  
  
  
    <property>  
        <name>yarn.resourcemanager.address.rm1</name>  
        <value>m7-psdc-kvm03:8032</value>  
    </property>  
    <property>  
        <name>yarn.resourcemanager.address.rm2</name>  
        <value>m7-psdc-kvm02:8032</value>  
    </property>  
  
  
    <property>  
        <name>yarn.resourcemanager.resource-tracker.address.rm1</name>  
        <value>m7-psdc-kvm03:8031</value>  
    </property>  
    <property>  
        <name>yarn.resourcemanager.resource-tracker.address.rm2</name>  
        <value>m7-psdc-kvm02:8031</value>  
    </property>  
  
  
    <property>  
        <name>yarn.resourcemanager.webapp.address.rm1</name>  
        <value>m7-psdc-kvm03:8088</value>  
    </property>  
    <property>  
        <name>yarn.resourcemanager.webapp.address.rm2</name>  
        <value>m7-psdc-kvm02:8088</value>  
    </property>  
  
  
    <property>  
        <name>yarn.resourcemanager.scheduler.address.rm1</name>  
        <value>m7-psdc-kvm03:8030</value>  
    </property>  
    <property>  
        <name>yarn.resourcemanager.scheduler.address.rm2</name>  
        <value>m7-psdc-kvm02:8030</value>  
    </property>  
  
  
       <property>  
        <name>yarn.nodemanager.aux-services</name>  
        <value>mapreduce_shuffle</value>  
        <description>默认</description>  
       </property>  
  
  
    <property>  
        <name>yarn.nodemanager.pmem-check-enabled</name>  
        <value>false</value>  
    </property>  
    <property>  
        <name>yarn.log-aggregation-enable</name>  
        <value>true</value>  
    </property>  
<!--  
    <property>  
        <name>yarn.resourcemanager.ha.id</name>  
        <value>rm1</value>  
    </property>  
-->  
    <property>  
        <name>yarn.nodemanager.delete.debug-delay-sec</name>  
        <value>86400</value>  
    </property>  
  
  
       <property>  
        <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>  
        <value>org.apache.hadoop.mapred.ShuffleHandler</value>  
       </property>  
       <property>  
        <name>yarn.nodemanager.resource.memory-mb</name>  
        <value>102400</value>  
     </property>  
    <property>  
        <name>yarn.nodemanager.resource.cpu-vcores</name>  
        <value>20</value>  
    </property>  
    <property>  
        <name>yarn.scheduler.maximum-allocation-mb</name>  
        <value>102400</value>  
    </property>  
</configuration>

ZooKeeper 配置

Hadoop 正常配置

Hadoop 集群启动

第一次配置启动

在三个节点上启动Journalnode deamons，然后jps，出现JournalNode进程。
```
sbin/./hadoop-daemon.sh start journalnode
jps
JournalNode
```
格式化master上的namenode（任意一个），然后启动该节点的namenode。
```
bin/hdfs namenode -format
sbin/hadoop-daemon.sh start namenode
```
在另一个namenode节点slave1上同步master上的元数据信息
```
bin/hdfs namenode -bootstrapStandby
```
停止hdfs上的所有服务
```
sbin/stop-dfs.sh
```
初始化zkfc
```
bin/hdfs zkfc -formatZK
```
启动hdfs
```
sbin/start-dfs.sh
```
启动yarn
```
sbin/start-yarn.sh
```

非第一次配置启动

直接启动hdfs和yarn即可，namenode、datanode、journalnode、DFSZKFailoverController都会自动启动。
```
sbin/start-dfs.sh
```
启动yarn
```
sbin/start-yarn.sh
```

运维知识体系

Fri, 20 Apr 2018 00:00:00 +0000

转载请注明来自于-运维社区：https://www.unixhot.com/

运维架构层级/运维角度			内容描述/主要技术关键词	监控体系	自动化/DevOps	云计算
客户端层	浏览器		Cookie、浏览器缓存协商（Last-Modified、Expires、Etag）、组件分离、前端优化、运维检测工具	舆论监控外部网络监控 APM	故障检测工具	DNS服务 CDN服务移动服务云盾
	DNS		浏览器DNS缓存、DNS缓存、自建DNS服务器、商业DNS产品、智能DNS、公共DNS（BGP anycast）、bind+DLZ/DPDK
	客户端/APP		HTTP-DNS、打点日志、加密传输、移动推送、各类SDK（监控SDK、推流SDK等）
外部层	第三方CDN		GSLB、反向代理缓存、分布式存储、流量调度、配置管理、用户端（各类API如：带宽监控、预缓存、缓存刷新）		基于开放API开发
	云计算		公有云服务、混合云、运维外包服务、APM（应用性能管理）、第三方安全解决方案（防DDOS、WAF）
网络层	互联层		多机房互联（VPN、专线）、异地灾备-->异地多活-->按SET部署	设备监控（Zabbix SNMP）网络质量监控（Smokeping）	SDN OpenvSwitch （GRE、Vxlan）	高速通道
	核心层		防火墙、路由器、Ipsec VPN、链路负载均衡和高可用（CCNP级别）			VPC（专有网络）
	汇聚层		三层交换动态路由（OSPF）、静态路由、EC（端口汇聚）、MSTP+VRRP等（CCNP级别）
	接入层		二层交换（VTP、SPF、Trunk、端口安全）等（CCNA级别）
接入层	负载均衡高可用	四层负载均衡	开源：LVS（IP负载均衡）+Keepalived、Haproxy 商业：F5、Netscaler	服务监控（API）	平台开发（LBaas）	高防IP 云负载均衡SLB CDN服务
		七层负载均衡	反向代理：Haproxy、Nginx、Apache（根据HTTP协议支持的属性进行L7分发）、A/B Test Gateway、WAF
	反向代理缓存		ATS、Squid、Varnish、Nginx(缓存分级、预缓存、缓存刷新）
应用服务层	Web服务层		HTTP协议、Web服务器（Apache、Nginx/OpenResty、Tomcat、Resin、Jboss）安全设置、性能优化	业务监控(API)流量分析(Piwik) 服务监控(API) 安全监控(WAF)	配置管理：SaltStack 过载保护-服务降级灰度发布-openresty 项目管理-Readmine 代码仓库-gitlab 持续集成-Jenkins 持续审查-SonarQube	镜像市场
	应用服务层		运行环境（PHP Python Java C C++）、性能优化、缓存（OPCache、LocalCache）、Session存储、代码部署			各种SAAS服务
	业务层	业务实现	API网关、302调度、业务模块化（电商例：用户、商品、购物车、结算中心、价格等服务）、微服务
		服务层	SOA框架（Dubbo）、微服务框架（Spring Cloud）、协议（RPC、RESTful）、框架安全、应用性能监控			分布式应用服务
	分布式层	消息队列	ActiveMQ（成熟）、RabbitMQ（成熟、案例多）、RocketMQ（业务应用）、Kafka（日志传输）、ZeroMQ（快）			消息队列服务
存储层	文件存储	单机存储	块存储 - 机械硬盘、SSD、文件系统（ext4、xfs）、LVM、tmpfs	系统监控软件自带监控	配置管理	云硬盘对象存储
		单机存储扩展	文件分发（多级分发）、文件同步（rsync、inotify）、DRBD、DAS（块存储）
		共享存储	文件存储 - NAS[NFS（Unix/Linux）]、FTP、SAN、iSCSI
		分布式存储	对象存储 - GlusterFS、MooseFS、Ceph、FastDFS（非对象存储）
	DAL	数据访问层	应用层分片、淘宝TDDL、开源：360（Atlas）、阿里（Cobar）、MyCat、MySQL-Proxy、根据业务开发			数据库服务
	数据存储	分布式缓存	Memcached、Redis（客户端分片、Redis Cluster、Twemproxy、Codis）	数据库监控	数据库运维平台	云数据库-RDS Mongodb、Redis Memcached OceanBase
		NoSQL	Redis、LevelDB（SSDB）、CouchDB、Mongodb、Couchbase 、Cassandra、TiDB（支持MySQL协议）
		时间序列	RRDTool、Graphite Whisper、OpenTSDB、InfluxDB、KairosDB、ElasticSearch
		RDBMS	MySQL（PXC集群、MHA）、Oracle（DG、OGG、RAC）、PostgreSQL、SqlServer、SQLite、DB2
		大数据	Hadoop生态圈（HDFS、Hive、Hbase、Zookeeper、Pig、Spark、Impala、Kudu）、Mahout智能推荐	服务监控	Ambari、CM	大数据服务
基础服务层	业务决策		灰度发布、服务降级、异地灾备、数据分析平台、智能扩容决策树（需要各层支持）	监控工具： Zabbix Nagios Cacti Open-Falcon Sensu	自动化工具（Puppet Chef SaltStack Ansible）	日志服务操作审计资源编排运维监控服务持续交付系统
	运维相关		项目管理（Redmine、Jira、知识库、Bugzilla、CodeReview）、工单系统、运维操作平台、监控平台
	应用相关		持续集成、日志收集平台（ELKStack）、自动化部署平台、Job管理（调度）平台、安全扫描平台
	系统相关		LDAP、内部DNS、DHCP、Mail、SMS、Gitlab、Yum仓库、操作审计（xenapp）、堡垒机
容器层	容器编排		Mesos（Marathon、Chronos）、Kubernetes（SKYDNS）、Docker Swarm、CoreOS（fleet）、OpenStack（Magnum）	Docker Stats cAdvisor DataDog Zabbix	Docker Swarm Mesos Kubernetes	容器服务
	容器和系统		容器：LXC、LXD、Docker、rkt、系统：CoreOS、Atomic、RancherOS
	网络和存储		网络：Calico、Flanel、Weave Net 存储：Ceph 镜像管理：Docker Registry、Harbor
操作系统层	CPU		CPU运行级别、使用率、上下文切换、运行队列、进程调度、系统调用、CPU管理（进程管理、taskset、intel VT-X）	mpstat、strace	虚拟化	公有云弹性计算产品
	内存		虚拟内存、SWAP换入换出、内存寻址、内存管理（Buffer Cache、HugePages、ksmd、EPT）	vmstat、free
	I/O（磁盘）		缺页中断、IOPS（顺序IO、随机IO）、IO管理（IO调度算法、virtio）、VFS	iostat、iotop
	I/O（网络）		TCP/IP（三次握手、四次挥手、状态转换、TCP队列）、IO模型、Bonding、Bridge、网络管理（iftop、tcpdump）	iftop
	内核/Shell		内核定制、内存参数优化、脚本编程（AWK、Sed、Shell、Python、PHP、Perl、Ruby、Lua）	系统监控
基础设施层	IAAS(基础设施即服务)		公有云、私有云（OpenStack/cloudstack+KVM/XEN、oVirt）、混合云	服务监控	配置管理
	硬件管理		硬件选型、配件更换、资产录入、系统安装（Cobbler）、标签化、Raid构建、远程控制（KVM、iDrac、ILO、IMM）	巡检、IPMI	IPMI、CMDB
	IDC托管		需求分析、IDC选型、网络测试、谈价格、签合同、设备采购（原厂vs渠道）、机柜和机位规划
运维产品化	基于DevOps产品思路		项目管理（类似Jira）、Bug管理、代码托管（类似Gitlab）、持续交付（类似Jenkins的构建、测试、部署）	监控平台、看板	软件定义数据中心	DevOps产品
	自动化运维产品思路		CMDB、ITSM管理系统（事件管理、问题管理、故障管理、工单系统）、作业平台、堡垒机、APM、私有云平台	监控平台	CI/CD系统	运维管理产品
运维服务化	OAAS		OAAS：Operations as a Service,运维咨询、运维托管、技术培训、应急处理、产品即服务、DevOps专家服务
测试和开发相关			运维协助：性能测试（TCPCopy、日志转换）、单机监控（nmon）、环境规划（开发、测试、预生产、生产）、CI（持续集成）、自动化部署
运维管理体系			运维管理必会：ITSM、ITIL V3、IT Service CMM、Six Sigma、DevOps Master、项目管理（PMBok）、架构层面（知识体系、运维方案、容量规划、灾备规划、服务降级）
运维发展趋势（个人理解）			打杂（小公司啥都干）->分层（应用运维、系统运维、基础运维、运维开发等）->场景化（分业务）->自动化（最终大家的目标都是自动化）
运维自动化发展趋势（个人理解）			标准化（文档化、流程化）->工具化（流程固化为工具）->Web化（平台化）->服务化（API化）->智能化（自动化）->产品化（服务化，云服务、运维创业）
备注：			1.本表格只体现和运维相关的内容；2.表格没有严格意义上的层级关系；3.持续更新中，由于每个层次内容多，只例举比较出名（重要/开源）的关键词； 4.运维人员要给自己划好知识边界！（横向\|纵向）5.转载请注明来自-运维社区https://www.unixhot.com/page/ops

Github pages和mixpanel 统计

Mon, 16 Apr 2018 00:00:00 +0000

简介

Jekyll 提供了功能强大的博客功能，但是还是缺乏用户统计的功能。缺的这一块当前github io Pages
考虑使用麻雀虽小五脏俱全的MixPanel。Jekyll 本身带有分析统计的框架，需要客制化代码后才能正常运作。

Git Hub 配置

_config.xml 配置Token 变量

analytics :
    provider : mixpanel # 设置provider 为 mixpanel
    gauges : 
        site_id : 'SITE ID'
    google : 
        tracking_id : 'UA-123-12'
    getclicky : 
      site_id : 
    mixpanel : 
        token : 'Mixpanel Token' # 配置Token
    piwik : 
        baseURL : 'myserver.tld/piwik' # Piwik installation address (without protocol)
        idsite : '1'                   # the id of the site on Piwik

修改analytics-providers/mixpanel

路径

_includes/JB/analytics-providers/mixpanel
内容

script type="text/javascript">
(function(e,a){if(!a.__SV){var b=window;try{var c,l,i,j=b.location,g=j.hash;c=function(a,b){return(l=a.match(RegExp(b+"=([^&]*)")))?l[1]:null};g&&c(g,"state")&&(i=JSON.parse(decodeURIComponent(c(g,"state"))),"mpeditor"===i.action&&(b.sessionStorage.setItem("_mpcehash",g),history.replaceState(i.desiredHash||"",e.title,j.pathname+j.search)))}catch(m){}var k,h;window.mixpanel=a;a._i=[];a.init=function(b,c,f){function e(b,a){var c=a.split(".");2==c.length&&(b=b[c[0]],a=c[1]);b[a]=function(){b.push([a].concat(Array.prototype.slice.call(arguments, 0)))}}var d=a;"undefined"!==typeof f?d=a[f]=[]:f="mixpanel";d.people=d.people||[];d.toString=function(b){var a="mixpanel";"mixpanel"!==f&&(a+="."+f);b||(a+=" (stub)");return a};d.people.toString=function(){return d.toString(1)+".people (stub)"};k="disable time_event track track_pageview track_links track_forms register register_once alias unregister identify name_tag set_config reset people.set people.set_once people.unset people.increment people.append people.union people.track_charge people.clear_charges people.delete_user".split(" "); for(h=0;h<k.length;h++)e(d,k[h]);a._i.push([b,c,f])};a.__SV=1.2;b=e.createElement("script");b.type="text/javascript";b.async=!0;b.src="undefined"!==typeof MIXPANEL_CUSTOM_LIB_URL?MIXPANEL_CUSTOM_LIB_URL:"file:"===e.location.protocol&&"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js".match(/^\/\//)?"https://cdn.mxpnl.com/libs/mixpanel-2-latest.min.js":"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js";c=e.getElementsByTagName("script")[0];c.parentNode.insertBefore(b,c)}})(document,window.mixpanel||[]); mixpanel.init("\\{\\{ site.JB.analytics.mixpanel.token\\}\\}");
</script>

修改 posts 的内容

_includes/themes/bootstrap-3/post.html

\\{\\% include JB/analytics \\%\\}
<script>
 mixpanel.track("View posts");
</script>

Mixpanel 统计截图

最大回文算法

Thu, 12 Apr 2018 00:00:00 +0000

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
public class Solution {

	static Solution sa = new Solution();

	char[] ss;
	int left = 0, max = 1;

	public String longestPalindrome(String s) {
		ss = s.toCharArray();
		if (ss.length < 1) return "";
		for (int i = 0; i < ss.length; i++) {
			i = update(i);
		}
		return s.substring(left, left + max);
	}

	private int update(int index) {
		int ll = index, rr = index;
		// ()->rr || []->ll
		// [ll,rr]->(a)aa    rr=index
		// [a](a)a           rr=index+1 传递
		// [a]a(a)           rr=index+2
		while (rr + 1 < ss.length && ss[rr] == ss[rr + 1]) {
			rr++;
		}
		int temp = rr;
		// ll-1 | rr+1，下一个匹配
		// [ll-1][a]a(a)(rr+1) rr=index+2
		while (rr + 1 < ss.length && ll - 1 >= 0 && ss[rr + 1] == ss[ll - 1]) {
			rr++;
			ll--;
		}
		if (rr - ll + 1 > max) {
			max = rr - ll + 1; // 计算长度
			left = ll; // 记录开始位置
		}
		return temp;
	}

	public static void main(String[] args) {
		System.out.println(sa.longestPalindrome("babad"));
	}

}

字符串查找以及edi报文

Thu, 12 Apr 2018 00:00:00 +0000

package ext.spring.smooks;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.milyn.Smooks;
import org.milyn.container.ExecutionContext;
import org.milyn.payload.StringResult;
import org.milyn.payload.StringSource;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

public class SmookEDITest {

	public static String parseStreamToString(InputStream in) {
		StringBuffer sb = new StringBuffer();
		if (in != null) {
			byte[] b = new byte[128];
			int c = -1;
			try {
				while ((c=in.read(b)) != -1) {
					for (int i = 0;i < c;i++) {
						sb.append((char)b[i]);
					}
				}
			} catch (Exception e) {
				e.printStackTrace();
			}

		}

		return sb.toString();
	}


	public static void testEdi2Smook(String cfgInStr, String orderInStr) {
		InputStream cfgIn = null;
		InputStream orderIn=null;
		try {
			cfgIn = new FileInputStream(cfgInStr);
			orderIn = new FileInputStream(orderInStr);
			String msgIn = SmookEDITest.parseStreamToString(orderIn);
			System.out.println(msgIn);
			Smooks smooks1 = new Smooks(cfgIn);

			/*
			 * Create a Smook Validator
			 */
			// SmooksResourceConfiguration pojoConf = new SmooksResourceConfiguration("", Profile.DEFAULT_PROFILE, "");

			ExecutionContext executionContext = smooks1.createExecutionContext();
			StringResult stringRs = new StringResult();
			smooks1.filterSource(executionContext, newStringSource(msgIn), stringRs);




			System.out.println(stringRs.getResult());

		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (cfgIn != null) {
				try {
					cfgIn.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			if (orderIn != null) {
				try {
					orderIn.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}   

	}

	private static XPathExpression EXP_TRANSACTION_TABLE= null;
	private static XPathExpression EXP_NODE_DATA_ELEMENT = null;

	static {
		XPathFactory factory = XPathFactory.newInstance();
		XPath xpath = factory.newXPath();

		try {
			EXP_TRANSACTION_TABLE = xpath.compile("/transactionSet/table");
			EXP_NODE_DATA_ELEMENT = xpath.compile("dataElement");
		} catch (Exception e) {
			e.printStackTrace();
		}

	}


	public Document transDoc(InputStream in) {
		Document doc = null;
		DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
		dbf.setNamespaceAware(true);
		dbf.setIgnoringElementContentWhitespace(true);

		try {
			DocumentBuilder db = dbf.newDocumentBuilder();
			doc = db.parse(in);
		} catch (Exception e) {
			e.printStackTrace();
		}
		return doc;
	}

	public void TranslateOBOE2Smook(String oboeIn) {
		File oboeCfgFile = new File(oboeIn);

		String tabs="\t";
		InputStream in = null;
		try {

			System.out.println(this.appendix);
			in = new FileInputStream(oboeCfgFile);
			Document doc = transDoc(in);
			NodeList topNodes = (NodeList) EXP_TRANSACTION_TABLE.evaluate(doc, XPathConstants.NODESET);
			int topLen = topNodes.getLength();
			System.out.println(tabs+"<medi:segmentGroup maxOccurs=\"1\" xmltag=\"" + "TransactionSet" + "\">");
			for (int i=0;i<topLen;i++) {
				//transactionSet
				Node tranNode = topNodes.item(i);
				// String tblSecName = tranNode.getAttributes().getNamedItem("section").getNodeValue();

				// Smook Group for 
				// Do loop
				NodeList segNodes = tranNode.getChildNodes();
				for (int j=0;j<segNodes.getLength();j++) {
					doLoop(segNodes.item(j),tabs+"\t", false, false);
				}
			}
			System.out.println(tabs+"</medi:segmentGroup>");
			System.out.println(this.suffix);
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (in != null) {
				try {
					in.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}


	}


	public static String mapAttrOboe2Smook(Node node, String oboeAttr, String smookAttr) {
		StringBuffer preSb = new StringBuffer();
		Node segCodeAttr = node.getAttributes().getNamedItem(oboeAttr);
		if (segCodeAttr != null) {
			preSb.append(" " + smookAttr + "=\"").append(segCodeAttr.getNodeValue()).append("\"");
		}
		return preSb.toString();
	}

	public static void doLoop(Node node, String tabs, booleanisTop, boolean isPreLoop) {
		String nodeName = node.getNodeName();
		if ("segment".equals(nodeName)) {
			String occurs = node.getAttributes().getNamedItem("occurs").getNodeValue();
			String segGroupOrSeg = "";

			StringBuffer preSb = new StringBuffer();


			String required = node.getAttributes().getNamedItem("required").getNodeValue();
			String toDup = "";
			if("M".equals(required)) {
				preSb.append(" minOccurs=\"1\"");
				toDup = " minOccurs=\"1\"";
			} else if ("O".equals(required)) {
				preSb.append(" minOccurs=\"0\"");
				toDup = " minOccurs=\"0\"";
			} else {
				System.out.println(required+"<!------------------------------------------------------------------------>");
			}

			if (isPreLoop) {
				segGroupOrSeg = "segment";
				preSb.append(mapAttrOboe2Smook(node, "id", "segcode")).append(mapAttrOboe2Smook(node, "xmlTag", "xmltag"));
				preSb.append(" truncatable=\"true\"");
			} else {
				if (!(occurs == null || "".equals(occurs))) {
					if ("1".equals(occurs)) {
						segGroupOrSeg = "segment";
						preSb.append(mapAttrOboe2Smook(node, "id", "segcode")).append(mapAttrOboe2Smook(node, "xmlTag", "xmltag"));
						preSb.append(" truncatable=\"true\"");
					} else {
						segGroupOrSeg = "segmentGroup";
						preSb.append(mapAttrOboe2Smook(node, "xmlTag", "xmltag")).append(mapAttrOboe2Smook(node, "occurs", "maxOccurs"));
					}
				} else {
					segGroupOrSeg = "segment";
					preSb.append(mapAttrOboe2Smook(node, "id", "segcode")).append(mapAttrOboe2Smook(node, "xmlTag", "xmltag"));
					preSb.append(" truncatable=\"true\"");
				}
			}
			System.out.println(tabs+"<medi:" + segGroupOrSeg + preSb.toString() + ">");

			NodeList dataEleNodes;
			try {
				dataEleNodes = (NodeList) EXP_NODE_DATA_ELEMENT.evaluate(node, XPathConstants.NODESET);
				if (!(dataEleNodes == null || dataEleNodes.getLength() < 1)) {

					if ("segmentGroup".equals(segGroupOrSeg)) {
						String tt = tabs+"<medi:segment " + mapAttrOboe2Smook(node, "xmlTag", "xmltag") + mapAttrOboe2Smook(node, "id", "segcode") + toDup + " truncatable=\"true\"" + ">";
						System.out.println(tt);
					}
					for (int i=0;i<dataEleNodes.getLength();i++) {
						// process data elements
						Node dataEleNode = dataEleNodes.item(i);
						if ("dataElement".equals(dataEleNode.getNodeName())) {
							System.out.println(tabs+"\t<medi:field xmltag=\"" + dataEleNode.getAttributes().getNamedItem("xmlTag").getNodeValue() + "\"/>");
						}
					}
					if ("segmentGroup".equals(segGroupOrSeg)) {
						System.out.println(tabs+"</medi:segment>");
					}
				}

			} catch (XPathExpressionException e) {
				e.printStackTrace();
			}

			System.out.println(tabs+"</medi:" + segGroupOrSeg + ">");
		} else if ("loop".equals(nodeName)) {


			String occurs = node.getAttributes().getNamedItem("occurs").getNodeValue();
			String xmltag = node.getAttributes().getNamedItem("xmlTag").getNodeValue();


			String required = node.getAttributes().getNamedItem("required").getNodeValue();
			String toDup = "";
			if("M".equals(required)) {
				toDup = " minOccurs=\"1\"";
			} else if ("O".equals(required)) {
				toDup = " minOccurs=\"0\"";
			} else {
				System.out.println(required+"<!------------------------------------------------------------------------>");
			}
			boolean isLoop = false;
			String tt ="";
			if (isTop) {

			} else {
				tt = tabs+"<medi:segmentGroup xmltag=\"" + xmltag + "\" maxOccurs=\"" + occurs + "\""+ toDup + ">";
				isLoop=true;
				//tt = tabs+"<medi:segment" + mapAttrOboe2Smook(node, "xmlTag", "xmltag") + mapAttrOboe2Smook(node, "id", "segcode") + " truncatable=\"true\"" + " maxOccurs=\"" + occurs + "\""+ toDup + ">";
			}
			System.out.println(tt);
			NodeList childNodes = node.getChildNodes();
			for(int i=0;i<childNodes.getLength();i++) {
				doLoop(childNodes.item(i),tabs+"\t", false, isLoop);

			}
			if (isTop) {

			} else {
				//System.out.println(tabs+"</medi:segment>");
				System.out.println(tabs+"</medi:segmentGroup>");
			}
		} else if ("compositeDE".equals(nodeName)) {
			String occurs = node.getAttributes().getNamedItem("occurs").getNodeValue();
			String xmltag = node.getAttributes().getNamedItem("xmlTag").getNodeValue();
			String required = node.getAttributes().getNamedItem("required").getNodeValue();

			Integer minLength = Integer.valueOf(node.getAttributes().getNamedItem("minLength").getNodeValue());
			Integer maxLength = Integer.valueOf(node.getAttributes().getNamedItem("maxLength").getNodeValue());
			boolean isRequired = false;
			if("M".equals(required)) {
				isRequired = true;
			} else if ("O".equals(required)) {
				isRequired = false;
			}
			if (isPreLoop) {
				System.out.println(tabs + "<medi:field xmltag=\"" + xmltag + "\" required=\"" + isRequired + "\" minLength=\"" + minLength + "\" maxLength=\"" + maxLength + "\"/>");
			} else {
				System.out.println(tabs+"<medi:component xmltag=\"" + xmltag+ "\" required=\"" + isRequired + "\" minLength=\"" + minLength + "\" maxLength=\"" + maxLength + "\">");
				NodeList childNodes = node.getChildNodes();
				for(int i=0;i<childNodes.getLength();i++) {
					doLoop(childNodes.item(i),tabs+"\t", false, true);
				}
			}
			System.out.println("</medi:component>");
		} else {
			// do nothing

		}
	}

	private String appendix="";
	private String suffix="";

	public void readAppendix(String fileloc) {
		InputStream in = null;
		try {
			in = new FileInputStream(fileloc);
			this.appendix = parseStreamToString(in);
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} finally {
			if (in != null) {
				try {
					in.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public void readSuffix(String fileloc) {
		InputStream in = null;
		try {
			in = new FileInputStream(fileloc);
			this.suffix = parseStreamToString(in);
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} finally {
			if (in != null) {
				try {
					in.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
	}





	public static int findChar(int c, String pattern, int step) {
		int strLen = pattern.length();
		++step;
		if (step < strLen) {
			if (pattern.charAt(step) == c) {
			} else {
				step = -1;
			}
		}
		return step;
	}


	public static void tryToFindUnknown(String strSource, String defination, String pattern) {


		Map<String, Boolean> segments = checkUnknownSegmentsPattern(strSource, pattern);
		for (String keyPattern : segments.keySet()) {
			System.out.println("Key:\t" + keyPattern + "\t\t\t ?Exists:\t" + findAsPattern(defination, "\"" + keyPattern + "\""));
		}
	}


	public static Map<String, Boolean> checkUnknownSegmentsPattern(String strSource, String pattern) {
		// search segments in ediFile
		Map<String, Boolean> result = new HashMap<String, Boolean>();
		FileReader reader = null;
		BufferedReader br = null;
		try {
			reader = new FileReader(strSource);
			br = new BufferedReader(reader);
			Pattern p = Pattern.compile(pattern);
			int c = -1;
			StringBuffer sb = new StringBuffer();
			while ((c = br.read()) != -1) {
				sb.append((char) c);
			}
			Matcher m = p.matcher(sb.toString());
			while (m.find()) {
				result.put(m.group().substring(1, m.group().length()-1), false);
			}
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			if (br != null) {
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
		return result;
	}


	public static boolean findAsPattern(String strSource, String pattern) {
		// search segments in ediFile
		FileReader reader = null;
		BufferedReader br = null;
		boolean rs =false;
		try {
			reader = new FileReader(strSource);
			br = new BufferedReader(reader);

			int c = -1;
			int step = -1;
			int count = 0;
			int pos = -1;
			StringBuffer sb = new StringBuffer();
			while ((c = br.read()) != -1) {
				pos++;
				count++;
				sb.append((char) c);
				step = findChar(c, pattern, step);
				if (step == pattern.length()-1) {
					step = -1;
					String ss =sb.toString().substring(1);
					pos-=ss.length();
					br.mark(pos);
					count = 0;
					sb = new StringBuffer();
					rs = true;
					break;
				} else if (step == -1 ) {
					if( count > 1) {
						String ss =sb.toString().substring(1);
						pos-=ss.length();
						br.mark(pos);
						count = 0;
					} else if (count == 1) {

					}
					sb = new StringBuffer();
				}
			}
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			if (br != null) {
				try {
					br.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			if (reader != null) {
				try {
					reader.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		}
		return rs;
	}


	public static void main(String[] args) {
		String regex = "~...?\\*";
		Pattern p =Pattern.compile(regex);
		String str = "ISA*00* *00* *ZZ*DHIWAL *....IEA*1*000000000~";
		Matcher m = p.matcher(str);
		while(m.find()) {
			System.out.println(m.group());
		}
		String ediPath = SmookEDITest.class.getResource("order.edi").getPath();
		String defPath = SmookEDITest.class.getResource("edi-mapping-856-test.xml").getPath();
		tryToFindUnknown(ediPath, defPath, regex);
		SmookEDITest t = new SmookEDITest();
		t.readAppendix(SmookEDITest.class.getResource("appendix").getPath());
		t.readSuffix(SmookEDITest.class.getResource("suffix").getPath());
		t.TranslateOBOE2Smook(SmookEDITest.class.getResource("856.xml").getPath());
		testEdi2Smook(SmookEDITest.class.getResource("edi-smook-config.xml").getPath(), SmookEDITest.class.getResource("oboe_order.edi").getPath());

	}

}

压力测试脚本

Thu, 12 Apr 2018 00:00:00 +0000

Main

#!/usr/local/bin/perl

#TODO Cofig below will be in configure.xml
$PROP_LOC=shift @ARGV;#Read Argument

printf "Stress Test Config file is:$PROP_LOC\n";
unless (open($fh, $PROP_LOC)) {#Open file handler
    die("Can't open $PROP_LOC");

};


%config = map /^\s*(\w+)\s*=\s*(.*)/, <$fh>;#Map Property File
($sec,$min,$hour,$day,$mon,$year,$wday,$yday,$isdst)=localtime(time());#Timestam Function

# variable settings
$TIME_OUT=$config{FileCopyTimeout};
$THREAD_COUNT = $config{ThreadCount};
$FILE_BATCH_SIZE=$config{FileMultiplier};
@TEST_DISTRIBUTORS=split(",",$config{TargetDistributorIDs});
@TEST_DIR=split(",",$config{TargetDirList});
$FILE_MESSAGE=$config{FileMessage};

$FILE_GEN_PATH=$config{GeneratePath};
#my $test_flag = validate_target(@TEST_DIR);
$GOOD_FILE = $config{GoodDataPath};
$TARGET_PER_HOUR=$config{TargetFilesPerHour};

# Calculate how many messasge to be sent within one thread
$MESSAGE_PER_THREAD=$TARGET_PER_HOUR/$THREAD_COUNT;

# Wating time for thread
$SLEEP_TIME=1/(3600*$MESSAGE_PER_THREAD);
# Indicate how many line of data in the sample good file
$FILE_LENGTH = $config{FileLength};
# Array of threads
my @WORKERS;

#while(my ($key,$val) =each %config) {
#};



# Initiate threads
for(my $i=0;$i<$THREAD_COUNT;$i++) {
    my $pid=fork();#Use fork
    unless($pid) {#Test PID created
        exec("perl stress_test_CXFTYPE_send_worker.pl '$PROP_LOC' $i”);#Must use exec
        exit;#Should exit
    }
}


# wait all child processs finished.
while(waitpid(-1, WUNTRACED)>0){sleep 1;};#Test if all pid is done, then exit

printf "Exit Stress Test.\n";

Worker

#!/usr/local/bin/perl

#TODO Cofig below will be in configure.xml

$PROP_LOC=shift @ARGV;
$Thread_Num=shift @ARGV;
printf "Worker===>Stress Test Config file is:$PROP_LOC\n";
unless (open($fh, $PROP_LOC)) {
        die("Can't open $PROP_LOC");

};


%config = map /^\s*(\w+)\s*=\s*(.*)/, <$fh>;


#%config=(
#   FileCopyTimeout     =>@STRESS_CXFTYPE_FILE_COPY_TIME_OUT@,
#   FileExtension       =>"@STRESS_CXFTYPE_FILE_EXTENSION@",
#   FileMultiplier      =>@STRESS_CXFTYPE_FILE_MULTIP@,
#   FileLength          =>@STRESS_CXFTYPE_FILE_LEN@,
#   FileMessage         =>"@STRESS_CXFTYPE_FILE_MSG@",
#   GoodDataPath        =>"@STRESS_CXFTYPE_GOOD_DATA_PATH@",
#   GeneratePath        =>"@STRESS_CXFTYPE_GEN_PATH@",
#   TargetDirList       =>"@STRESS_CXFTYPE_TARGET_DIR_LIST@",
#   #TargetDirList      =>"/ftpusers/web/delta/inbound/,/ftpusers/web/custname1/inbound/,/ftpusers/web/custname2/inbound/,/ftpusers/web/custname3/inbound/",
#   TargetDistributorIDs=>"@STRESS_CXFTYPE_TARGET_DISTRIBUTOR_IDS@",
#   TargetFilesPerHour  =>@STRESS_CXFTYPE_TARGET_FILES_PERHOUR@,
#   TargetHost          =>"@STRESS_CXFTYPE_TARGET_HOST@",
#   Username            =>"@STRESS_CXFTYPE_USR@",
#   TargetPort          =>@STRESS_CXFTYPE_PORT@,
#   ThreadCount         =>@STRESS_CXFTYPE_THREAD_COUNT@
#);


($sec,$min,$hour,$day,$mon,$year,$wday,$yday,$isdst)=localtime(time());

# variable settings
$TIME_OUT=$config{FileCopyTimeout};
$THREAD_COUNT = $config{ThreadCount};
$FILE_BATCH_SIZE=$config{FileMultiplier};
@TEST_DISTRIBUTORS=split(",",$config{TargetDistributorIDs});
@TEST_DIR=split(",",$config{TargetDirList});
$FILE_MESSAGE=$config{FileMessage};

$FILE_GEN_PATH=$config{GeneratePath};
#my $test_flag = validate_target(@TEST_DIR);
$GOOD_FILE = $config{GoodDataPath};
$TARGET_PER_HOUR=$config{TargetFilesPerHour};

# Calculate how many messasge to be sent within one thread
$MESSAGE_PER_THREAD=$TARGET_PER_HOUR/$THREAD_COUNT;

# Wating time for thread
$SLEEP_TIME=1/(3600*$MESSAGE_PER_THREAD);
# Indicate how many line of data in the sample good file
$FILE_LENGTH = $config{FileLength};
# Array of threads

#SCP command function
$SCP_CMD="scp -o ConnectTimeout=$TIME_OUT";
$SSH_CMD="ssh -o NumberOfPasswordPrompts=0  -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=quiet ";

$CP_CMD="cp ";
sub sendFile($$) {
    my $src = $_[0];
    my $dest= $_[1];
    my $full_cmd = "$SCP_CMD $src $config{Username}\@$config{TargetHost}:$dest";
    print $full_cmd;
    system($full_cmd);
}


sub copyFile($$) {# $$ Means String String
    my $src=$_[0];
    my $dest= $_[1];
    my $full_cmd = $CP_CMD."$src $dest";
    print $full_cmd;
    system($full_cmd);
}

sub validate_target(@@) {
    my $full_cmd = "$SSH_CMD $config{Username}\@config{TargetHost}";

}

sub myRand ($){
    my $num_len = $_[0];
    my $rnd_num=int(rand(10**$num_len - 1));
    if ($rnd_num<0) {
        $rnd_num++;
    }
    my $num_str="".$rnd_num;
    my $zero_append_num = $num_len - length($num_str);
    if ($zero_append_num > 0) {
        for (my $i=0;$i<$zero_append_num;$i++) {
            $num_str="0".$num_str;
        }
    }
    return $num_str;

}





# sender threads
sub work_func_sender{
    my $distributor_count=@TEST_DISTRIBUTORS;
    my $distri_idx=0;
    my $message_counter=0;
    printf "Message per Thread => $MESSAGE_PER_THREAD\n";
    while($message_counter<$MESSAGE_PER_THREAD) {
        $cur_distri_id = @TEST_DISTRIBUTORS[$distri_idx];
        $cur_dir = @TEST_DIR[$distri_idx];
        # CUST_Tracking_310_20150608_183000_123456.wff
        my @new_filename_arr;

        # Calc file(message) to send
        my $delta_message=$MESSAGE_PER_THREAD-$message_counter;
        if($delta_message >= $FILE_BATCH_SIZE) {
            $delta_message=$FILE_BATCH_SIZE;
        }
        printf ("File Multi $delta_message\n");
        for (my $i=0;$i<$delta_message;$i++) {
            print "Thread $Thread_Num i is $i\n";
            my $new_filename=$FILE_MESSAGE."_".$cur_distri_id."_"."$year$mon$day_$hour$min$sec"."_".myRand(6).".".$config{FileExtension};
            my $src_file=$FILE_GEN_PATH.$new_filename;
            copyFile($GOOD_FILE,$src_file);
            # push to new_filename_arr
            push(@new_filename_arr,$src_file);
        }
        # Begin write Data

        # End Write Data
        printf "Gernerate path: $src_file";
        my $dest_file=$cur_dir;
        # execute send
        $src_file =join(' ', @new_filename_arr);
        sendFile($src_file, $dest_file);
        $message_counter+=$delta_message;
        $distri_idx++;
        $distri_idx = int($distri_idx % $distributor_count);
        # Sleep time
        sleep($SLEEP_TIME);
    }
}


work_func_sender();

Zookeeper索引

Thu, 12 Apr 2018 00:00:00 +0000

ZooKeeper

启动

配置文件

# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial 
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between 
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just 
# example sakes.
dataDir=/var/applications/zk_test/data1
# the port at which the clients will connect
clientPort=2181
server.1=127.0.0.1:2221:3222
server.2=127.0.0.1:2222:3222
server.3=127.0.0.1:2223:3223
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the 
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1

启动命令

bin/zkServer.sh start conf/zoo1.cfg

连接客户端

bin/zkCli.sh -server 127.0.0.1:2181
#如果遇到无法连接，尝试修改iptables
-A INPUT -m state –state NEW -m tcp -p tcp –dport 2181 -j ACCEPT

Administration

Cross Machine

为了部署一个能接受 F 台失败机器的集成环境，总共需要总共2*F+1 台机器，所以通常机器数量是 Odd Number

Maintainance

Ongoing Data Cleanup

ZooKeeper 不会移除old snapshots 和 log files.
PurgeTxnLog实现了基本的 Retention Policy

java -cp zookeeper.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.15.jar:conf org.apache.zookeeper.server.PurgeTxnLog -n
3.4.0 后使用 autopurge.snapRetainCount 和 autopurge.purgeInterval 参数实现自动清理, Advance Configuration

Monitoring

4word Command

echo `srst|conf|4word_cmd` |nc localhost 2181

避免的坑

inconsistent lists of servers
incorrect placement of transaction log
incorrect Java heap size
- ZooKeeper jvm设置成 no swap to disk
- 4G 的内存，不要设置 4G 及以上的 jvm max heap size
publicly accessible deployment
- 部署在防火墙内
可恢复的重发机制
tckTime 的设置

ZooKeeper Waters

getData(), getChildren(), and exists()

一次性的 watchEvent Trigger

举例子：/zone1 一次修改后，客户端得到watchEvent Trigger。
/zone1 再一次修改后，除非客户端完成read（这会设置一个watcher）
客户端看到的信息是order 保证的
Events

Created, Deleted, Changed, Child
Watches 机制保证
- ZooKeeper Client Lib 保证一切顺序派发
- node 的 watch event 于真正获取该 node 值之前发生

ZooKeeper ACL 控制

Pluggable ZooKeeper authentication

authProvider.1=com.f.MyAuth

public interface AuthenticationProvider {
String getScheme();
KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte authData[]);
boolean isValid(String id);
boolean matches(String id, String aclExpr);
boolean isAuthenticated();
}

一致性保证

Sequential Consistency
Updates from a client will be applied in the order that they were sent.
Atomicity
Updates either succeed or fail – there are no partial results.
Single System Image
A client will see the same view of the service regardless of the server that it connects to.
Reliability
1. If a client gets a successful return code, the update will have been applied. On some failures (communication errors, timeouts, etc) the client will not know if the update has applied or not. We take steps to minimize the failures, but the guarantee is only present with successful return codes. (This is called the monotonicity condition in Paxos.)

Any updates that are seen by the client, through a read request or successful update, will never be rolled back when recovering from server failures.

Timeliness
The clients view of the system is guaranteed to be up-to-date within a certain time bound (on the order of tens of seconds). Either system changes will be seen by a client within this bound, or the client will detect a service outage.

Mcsnode chart

Sat, 07 Apr 2018 00:00:00 +0000

时序解析

线程顺序

Thread	Predecossor	Currrent	Next
T1	P=null » P=C1	C1	C1.next =NULL
T2	P=C1	C2	C1.next = C2
T3	P=C2	C3	C2.next = C3

代码详解

T1 & T2
def Lock {
    P = GAS(C1) // P == NULL
}

def Unlock {
    if (c1.next == null) {
        if CAS(C1, NULL) { // T2 情况
            // 确实C1.next == null，因为如果执行过
            // T2 Lock: P = GAS(C2)
            // P = C2, 则CAS(C1, NULL)失败
            return; // 未执行 T2 Lock: P = GAS(C2)
        } else { // T2 情况
            // 确实C1.next == null，因为如果执行过
            // T2 Lock: P = GAS(C2)
            // P = C2, 则CAS(C1, NULL)失败
            while (C1.next == null) { // T2 情况
                // 等待next 挂载 C1.next = C2
            }
        }
    }
    // C1.next != null
    C1.next.isBlock = false; // 执行 T2 的 lock
    c1.next = null // GC
}

Static 和 volatile

Sun, 01 Apr 2018 00:00:00 +0000

图上纠正：staticInt 会在不同线程同步，时机不确定，但是忙碌时候另外一个线程只能用本线程 staticInt 的copy

static 静态部分在类加载的时候会在类对象保留一份，此类的实例间共享一份静态资源，并加载到内存，寄存器，缓存，不保证每个线程看到的值是一致，最新值（假设有线程对静态变量修改的话）；而volatile表示每次保证读取到到是主内存中最新的值。
现在JVM经过优化，已不会出现liveness failure 。所以没事别用volatile。1.7前知名bug

原始代码	优化代码
`while(!flag){}`	`if(flag){ &nbps;&nbps;&nbps;&nbps;while(true){} }`

死循环, 无volatile 关键字

public class TestWithoutVolatile {
    private static boolean bChanged;

    public static void main(String[] args) throws InterruptedException {
        new Thread() {
            @Override
            public void run() {
                for (;;) {
					if (bChanged == !bChanged) {
                        System.out.println("!=");
                        System.exit(0);
                    }
                }
            }
        }.start();
        Thread.sleep(300);
        new Thread() {
            @Override
            public void run() {
                for (;;) {
                    bChanged = !bChanged;
                }
            }
        }.start();
    }
}

有volatile，正确感知

public class TestWithVolatile {
    private static volatile boolean bChanged;
    public static void main(String[] args) throws InterruptedException {
        new Thread() {
            @Override
            public void run() {
                for (;;) {
                    if (bChanged == !bChanged) {
                        System.out.println("!=");
                        System.exit(0);
                    }
                }
            }
        }.start();
        Thread.sleep(100);
        new Thread() {
            @Override
            public void run() {
                for (;;) {
                    bChanged = !bChanged;
                }
            }
        }.start();
    }
}

测试代码

public class NoVisiblility {
    private static boolean ready;
    private static volatile boolean ready;// if $ready is volatile, this can be avoid as well even without $LINE 1
    private static int number;
    private static int cc;


    private static class ReaderThread extends Thread {
        public void run() { // 读线程
            while (!ready) {
                // $LINE 1
                System.out.println(222);// if this is removed, the copy of $ready will be used for ever
            }
            System.out.println(number);
            System.exit(0);
        }

    }

    private static class ReaderThread extends Thread {
        public void run() { // 读线程
            System.out.println("in");
            for (;;) {//  等价 while(!ready)
                // boolean aa = ready; 即使加入本变量 $aa 的读，也还是读取本地static copy
               // cc = 1; 即使加入其他变量的读写，也还是读取本地static copy
                // 如果执行ready = false; 就会同步$ready 在主存中的值，系统执行$LINE 2
                 if (ready) {
                    break; // $LINE 2
                }
            }
            System.out.println(number);
            System.out.println("exit");
            System.exit(0);
        }

    }


    public static void main(String[] args) throws InterruptedException { // 主线程
        new ReaderThread().start();
        Thread.sleep(10);
        new Thread(new Runnable() {

            @Override
            public void run() {
                for (;;) {
                    ready = true;
                    number = 42;
                }
            }
        }).start();;
    }
}

Jmx security

Sun, 01 Apr 2018 00:00:00 +0000

JVM 启动配置

-Dcom.sun.management.jmxremote 
-Dcom.sun.management.jmxremote.port=1234

配置用户及权限对应信息

${JAVA_HOME}/jre/lib/management/jmxremote.access

somebody readwrite
monitorRole readonly
controlRole readwrite

User Informatio Config

${JAVA_HOME}/jre/lib/management/jmxremote.password.template

somebody somebody_psw
monitorRole 123456
controlRole 123456

启动生效

-Dcom.sun.management.jmxremote 
-Dcom.sun.management.jmxremote.port=1234
 
-Dcom.sun.management.jmxremote.authenticate=true
-Dcom.sun.management.jmxremote.password.file=jmxpassword
-Dcom.sun.management.jmxremote.access.file=jmxaccess