HBASE的预分区设计

  hbase生成分区种子  

  1 package com.spdbccc.edm.storm.common;
  2 
  3 import java.util.List;
  4 
  5 import org.apache.commons.lang3.StringUtils;
  6 import org.apache.hadoop.hbase.util.Bytes;
  7 
  8 import com.google.common.collect.Lists;
  9 
 10 /**
 11  * HBase表管理器, 对Hbase表的rowkey规则生成,表名规则定义
 12  * 
 13  * @author WJ
 14  *
 15  */
 16 public class HTableManager {
 17 
 18     public static final byte[] DEFAULT_FAMILY_NAME = Bytes.toBytes("f1");
 19 
 20     private static final String[] PARTITIONS = generatPartitionSeed();
 21 
 22     /**
 23      * 生成3844个分区种子
 24      * 
 25      * @return String[]
 26      */
 27     public static String[] generatPartitionSeed() {
 28         List<Character> seeds = Lists.newArrayList();
 29         for (int i = '0'; i <= '9'; i++) {
 30             seeds.add((char) i);
 31         }
 32         for (int i = 'A'; i <= 'Z'; i++) {
 33             seeds.add((char) i);
 34         }
 35         for (int i = 'a'; i <= 'z'; i++) {
 36             seeds.add((char) i);
 37         }
 38         int k = 0;
 39         String[] partions = new String[seeds.size() * seeds.size()];
 40         for (int i = 0; i < seeds.size(); i++) {
 41             for (int j = 0; j < seeds.size(); j++) {
 42                 partions[k] = StringUtils.join(seeds.get(i), seeds.get(j));
 43                 k++;
 44             }
 45         }
 46         return partions;
 47     }
 48 
 49     /**
 50      * 按指定数量生成分区种子
 51      * 
 52      * @param limit
 53      * @return String[]
 54      */
 55     public static String[] generatPartitionSeed(int limit) {
 56         int size = PARTITIONS.length;
 57         int[] space = new int[limit];
 58         for (int pt = 0; pt < size;) {
 59             for (int j = 0; j < space.length; j++) {
 60                 ++space[j];
 61                 pt++;
 62                 if (pt == size) {
 63                     break;
 64                 }
 65             }
 66         }
 67         String[] seed = new String[limit + 1];
 68         int position = 0;
 69         for (int i = 0; i < space.length; i++) {
 70             seed[i] = PARTITIONS[position];
 71             position += space[i];
 72         }
 73         seed[seed.length - 1] = PARTITIONS[PARTITIONS.length - 1];
 74         return seed;
 75     }
 76 
 77     public static String generatRowkey(String str) {
 78         int i = Math.abs(str.hashCode() % PARTITIONS.length);
 79         return StringUtils.join(PARTITIONS[i], "-", str);
 80     }
 81 
 82     public static byte[] generatByteRowkey(String str) {
 83         int i = Math.abs(str.hashCode() % PARTITIONS.length);
 84         return Bytes.toBytes(StringUtils.join(PARTITIONS[i], "-", str));
 85     }
 86 
 87     public static String getEventLogTableName(String event) {
 88         return StringUtils.join("EVENT_LOG_", event);
 89     }
 90     
 91     public static String getGroupVarTableName() {
 92         return "CUSTOM_VARIABLE_GROUP";
 93     }
 94 
 95     
 96     public static String getActivityVarTableName() {
 97         return "CUSTOM_VARIABLE_ACTIVITY";
 98     }
 99 
100     public static String getUserVarTableName() {
101         return "CUSTOM_VARIABLE_USER";
102     }
103 
104     public static String getUserInfoTableName() {
105         return "USER_WIDE_PUB";
106     }
107 
108     public static String getCardInfoTableName() {
109         return "CARD_WIDE_PUB";
110     }
111 
112     public static String getAcctInfoTableName() {
113         return "ACCT_WIDE_PUB";
114     }
115 
116     public static String getMetricTableName() {
117         return "METRICS";
118     }
119 
120     public static String getCustDefinitionTableName() {
121         return "CUST_USER_DEFINITION";
122     }
123 
124     public static String getCardDefinitionTableName() {
125         return "CARD_USER_DEFINITION";
126     }
127     
128     public static String getEventLogLbsHisName(){
129         return "EVENT_LOG_LBS_HIS";
130     }
131     
132     public static String getRankLogName(){
133         return "RANK_LOG";
134     }
135 
136      public static void main(String[] args) {
137      String[] arr = generatPartitionSeed(101);
138      for (int i = 0; i < arr.length; i++) {
139      System.out.println(arr[i]);
140      }
141      }
142 }

按指定分区数量创建预分区表

 1 package com.spdbccc.edm.storm.common;
 2 
 3 import org.apache.hadoop.conf.Configuration;
 4 import org.apache.hadoop.hbase.HBaseConfiguration;
 5 import org.apache.hadoop.hbase.HColumnDescriptor;
 6 import org.apache.hadoop.hbase.HTableDescriptor;
 7 import org.apache.hadoop.hbase.TableName;
 8 import org.apache.hadoop.hbase.client.HBaseAdmin;
 9 import org.apache.hadoop.hbase.io.compress.Compression;
10 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
11 import org.apache.hadoop.hbase.util.Bytes;
12 
13 public class HTableUtil {
14 
15     private static final int MAX_FILE_SIZE = 1024 * 1024 * 256;
16 
17     public static void main(String[] args) throws Exception {
18         int limit = 100;
19 
20         createHBaseTable(HTableManager.getEventLogTableName("TEST"), limit);
21 
22         // createHBaseTable(HTableManager.getEventLogTableName("DH"), limit);
23 
24         // createHBaseTable(HTableManager.getActivityVarTableName(), limit);
25 
26         // createHBaseTable(HTableManager.getUserVarTableName(), limit);
27 
28     }
29 
30     private static HTableDescriptor getHTableDescriptor(String tableName) {
31         HColumnDescriptor columnDescriptor = new HColumnDescriptor(HTableManager.DEFAULT_FAMILY_NAME);
32         columnDescriptor.setCompressionType(Compression.Algorithm.SNAPPY);
33         columnDescriptor.setCompactionCompressionType(Compression.Algorithm.SNAPPY);
34         // columnDescriptor.setTimeToLive(60 * 60 * 24 * 365 * 1);
35         columnDescriptor.setBlockCacheEnabled(true);
36         columnDescriptor.setDataBlockEncoding(DataBlockEncoding.NONE);
37 
38         HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
39         desc.setCompactionEnabled(true);
40         desc.setMaxFileSize(MAX_FILE_SIZE);
41         desc.addFamily(columnDescriptor);
42         return desc;
43     }
44 
45     /**
46      * 创建表
47      * 
48      * @param tableName
49      *            表名
50      * @param partitionSeedLimit
51      *            预分区数量，最大数量3844
52      * @throws Exception
53      */
54     public static void createHBaseTable(String tableName, int partitionSeedLimit) throws Exception {
55         if (partitionSeedLimit > 3844 || partitionSeedLimit < 1) {
56             throw new IllegalArgumentException("PartitionSeedLimit must be > 0 and < 3844.");
57         }
58         System.out.println("init HBase admin...");
59         Configuration conf = HBaseConfiguration.create();
60         HBaseAdmin admin = new HBaseAdmin(conf);
61 
62         if (admin.tableExists(tableName)) {
63             if (!admin.isTableDisabled(tableName)) {
64                 admin.disableTable(tableName);
65             }
66             admin.deleteTable(tableName);
67             System.out.println(String.format("Table is exist, drop table %s successed.", tableName));
68         }
69 
70         System.out.println(String.format("Creating HBase table %s, partition seed limit %d.", tableName, partitionSeedLimit));
71         admin.createTable(getHTableDescriptor(tableName), Bytes.toByteArrays(HTableManager.generatPartitionSeed(partitionSeedLimit)));
72         System.out.println(String.format("HBase table %s is created.", tableName));
73         admin.close();
74         System.out.println("==============================================");
75     }
76 
77     // public static void main(String[] args) throws Exception {
78     // String tableName = HTableManager.getEventLogTableName(args[0]);
79     // int limit = Integer.parseInt(args[1]);
80     // System.out.println("创建表: " + tableName);
81     // createHBaseTable(tableName, limit);
82     // System.out.println("创建成功: " + tableName);
83     // }
84 
85 }

生成符合规则的rowkey
1 String key = "NE123456789";
2 String messageKey = HTableManager.generatRowkey(key) + "-" + YZYT_msg_type;