Apache HBase ™ Reference Guide

相似文件 换一批

是 否 +2 论坛币

k人 参与回答

经管之家送您一份

应届毕业生专属福利!

求职就业群

赵安豆老师微信：zhaoandou666

经管之家联合CDA

送您一个全额奖学金名额~ !

立即领取

感谢您参与论坛问题回答

经管之家送您两个论坛币！

+2 论坛币

本帖隐藏的内容

Apache HBase Reference Guide.pdf (13.9 MB)

2017-5-29 00:26:09 上传

扫码加我 拉你入群

请注明：姓名-公司-职位

以便审核进群资格，未注明则拒绝

分享0 收藏0 回帖

关键词：Reference erence apache Guide refer

1. Example 40. Create, modify and delete a Table Using Java
   
2. package com.example.hbase.admin;
   
3. 
   
4. import java.io.IOException;
   
5. 
   
6. import org.apache.hadoop.conf.Configuration;
   
7. import org.apache.hadoop.fs.Path;
   
8. import org.apache.hadoop.hbase.HBaseConfiguration;
   
9. import org.apache.hadoop.hbase.HColumnDescriptor;
   
10. import org.apache.hadoop.hbase.HConstants;
    
11. import org.apache.hadoop.hbase.HTableDescriptor;
    
12. import org.apache.hadoop.hbase.TableName;
    
13. import org.apache.hadoop.hbase.client.Admin;
    
14. import org.apache.hadoop.hbase.client.Connection;
    
15. import org.apache.hadoop.hbase.client.ConnectionFactory;
    
16. import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
    
17. 
    
18. public class Example {
    
19. 
    
20.   private static final String TABLE_NAME = "MY_TABLE_NAME_TOO";
    
21.   private static final String CF_DEFAULT = "DEFAULT_COLUMN_FAMILY";
    
22. 
    
23.   public static void createOrOverwrite(Admin admin, HTableDescriptor table) throws IOException {
    
24.     if (admin.tableExists(table.getTableName())) {
    
25.       admin.disableTable(table.getTableName());
    
26.       admin.deleteTable(table.getTableName());
    
27.     }
    
28.     admin.createTable(table);
    
29.   }
    
30. 
    
31.   public static void createSchemaTables(Configuration config) throws IOException {
    
32.     try (Connection connection = ConnectionFactory.createConnection(config);
    
33.          Admin admin = connection.getAdmin()) {
    
34. 
    
35.       HTableDescriptor table = new HTableDescriptor(TableName.valueOf(TABLE_NAME));
    
36.       table.addFamily(new HColumnDescriptor(CF_DEFAULT).setCompressionType(Algorithm.NONE));
    
37. 
    
38.       System.out.print("Creating table. ");
    
39.       createOrOverwrite(admin, table);
    
40.       System.out.println(" Done.");
    
41.     }
    
42.   }
    
43. 
    
44.   public static void modifySchema (Configuration config) throws IOException {
    
45.     try (Connection connection = ConnectionFactory.createConnection(config);
    
46.          Admin admin = connection.getAdmin()) {
    
47. 
    
48.       TableName tableName = TableName.valueOf(TABLE_NAME);
    
49.       if (!admin.tableExists(tableName)) {
    
50.         System.out.println("Table does not exist.");
    
51.         System.exit(-1);
    
52.       }
    
53. 
    
54.       HTableDescriptor table = admin.getTableDescriptor(tableName);
    
55. 
    
56.       // Update existing table
    
57.       HColumnDescriptor newColumn = new HColumnDescriptor("NEWCF");
    
58.       newColumn.setCompactionCompressionType(Algorithm.GZ);
    
59.       newColumn.setMaxVersions(HConstants.ALL_VERSIONS);
    
60.       admin.addColumn(tableName, newColumn);
    
61. 
    
62.       // Update existing column family
    
63.       HColumnDescriptor existingColumn = new HColumnDescriptor(CF_DEFAULT);
    
64.       existingColumn.setCompactionCompressionType(Algorithm.GZ);
    
65.       existingColumn.setMaxVersions(HConstants.ALL_VERSIONS);
    
66.       table.modifyFamily(existingColumn);
    
67.       admin.modifyTable(tableName, table);
    
68. 
    
69.       // Disable an existing table
    
70.       admin.disableTable(tableName);
    
71. 
    
72.       // Delete an existing column family
    
73.       admin.deleteColumn(tableName, CF_DEFAULT.getBytes("UTF-8"));
    
74. 
    
75.       // Delete a table (Need to be disabled first)
    
76.       admin.deleteTable(tableName);
    
77.     }
    
78.   }
    
79. 
    
80.   public static void main(String... args) throws IOException {
    
81.     Configuration config = HBaseConfiguration.create();
    
82. 
    
83.     //Add any necessary configuration files (hbase-site.xml, core-site.xml)
    
84.     config.addResource(new Path(System.getenv("HBASE_CONF_DIR"), "hbase-site.xml"));
    
85.     config.addResource(new Path(System.getenv("HADOOP_CONF_DIR"), "core-site.xml"));
    
86.     createSchemaTables(config);
    
87.     modifySchema(config);
    
88.   }
    
89. }

复制代码

1. Example 46. HBaseContext Usage Example
   
2. This example shows how HBaseContext can be used to do a foreachPartition on a RDD in Scala:
   
3. 
   
4. val sc = new SparkContext("local", "test")
   
5. val config = new HBaseConfiguration()
   
6. 
   
7. ...
   
8. 
   
9. val hbaseContext = new HBaseContext(sc, config)
   
10. 
    
11. rdd.hbaseForeachPartition(hbaseContext, (it, conn) => {
    
12. val bufferedMutator = conn.getBufferedMutator(TableName.valueOf("t1"))
    
13. it.foreach((putRecord) => {
    
14. . val put = new Put(putRecord._1)
    
15. . putRecord._2.foreach((putValue) => put.addColumn(putValue._1, putValue._2, putValue._3))
    
16. . bufferedMutator.mutate(put)
    
17. })
    
18. bufferedMutator.flush()
    
19. bufferedMutator.close()
    
20. })

复制代码

1. Example 47. bulkPut Example with DStreams
   
2. Below is an example of bulkPut with DStreams. It is very close in feel to the RDD bulk put.
   
3. 
   
4. val sc = new SparkContext("local", "test")
   
5. val config = new HBaseConfiguration()
   
6. 
   
7. val hbaseContext = new HBaseContext(sc, config)
   
8. val ssc = new StreamingContext(sc, Milliseconds(200))
   
9. 
   
10. val rdd1 = ...
    
11. val rdd2 = ...
    
12. 
    
13. val queue = mutable.Queue[RDD[(Array[Byte], Array[(Array[Byte],
    
14.     Array[Byte], Array[Byte])])]]()
    
15. 
    
16. queue += rdd1
    
17. queue += rdd2
    
18. 
    
19. val dStream = ssc.queueStream(queue)
    
20. 
    
21. dStream.hbaseBulkPut(
    
22.   hbaseContext,
    
23.   TableName.valueOf(tableName),
    
24.   (putRecord) => {
    
25.    val put = new Put(putRecord._1)
    
26.    putRecord._2.foreach((putValue) => put.addColumn(putValue._1, putValue._2, putValue._3))
    
27.    put
    
28.   })

复制代码

1. Example 48. Bulk Loading Example
   
2. The following example shows bulk loading with Spark.
   
3. 
   
4. val sc = new SparkContext("local", "test")
   
5. val config = new HBaseConfiguration()
   
6. 
   
7. val hbaseContext = new HBaseContext(sc, config)
   
8. 
   
9. val stagingFolder = ...
   
10. val rdd = sc.parallelize(Array(
    
11.       (Bytes.toBytes("1"),
    
12.         (Bytes.toBytes(columnFamily1), Bytes.toBytes("a"), Bytes.toBytes("foo1"))),
    
13.       (Bytes.toBytes("3"),
    
14.         (Bytes.toBytes(columnFamily1), Bytes.toBytes("b"), Bytes.toBytes("foo2.b"))), ...
    
15. 
    
16. rdd.hbaseBulkLoad(TableName.valueOf(tableName),
    
17.   t => {
    
18.    val rowKey = t._1
    
19.    val family:Array[Byte] = t._2(0)._1
    
20.    val qualifier = t._2(0)._2
    
21.    val value = t._2(0)._3
    
22. 
    
23.    val keyFamilyQualifier= new KeyFamilyQualifier(rowKey, family, qualifier)
    
24. 
    
25.    Seq((keyFamilyQualifier, value)).iterator
    
26.   },
    
27.   stagingFolder.getPath)
    
28. 
    
29. val load = new LoadIncrementalHFiles(config)
    
30. load.doBulkLoad(new Path(stagingFolder.getPath),
    
31.   conn.getAdmin, table, conn.getRegionLocator(TableName.valueOf(tableName)))

复制代码

1. Example 49. Using Additional Parameters
   
2. val sc = new SparkContext("local", "test")
   
3. val config = new HBaseConfiguration()
   
4. 
   
5. val hbaseContext = new HBaseContext(sc, config)
   
6. 
   
7. val stagingFolder = ...
   
8. val rdd = sc.parallelize(Array(
   
9.       (Bytes.toBytes("1"),
   
10.         (Bytes.toBytes(columnFamily1), Bytes.toBytes("a"), Bytes.toBytes("foo1"))),
    
11.       (Bytes.toBytes("3"),
    
12.         (Bytes.toBytes(columnFamily1), Bytes.toBytes("b"), Bytes.toBytes("foo2.b"))), ...
    
13. 
    
14. val familyHBaseWriterOptions = new java.util.HashMap[Array[Byte], FamilyHFileWriteOptions]
    
15. val f1Options = new FamilyHFileWriteOptions("GZ", "ROW", 128, "PREFIX")
    
16. 
    
17. familyHBaseWriterOptions.put(Bytes.toBytes("columnFamily1"), f1Options)
    
18. 
    
19. rdd.hbaseBulkLoad(TableName.valueOf(tableName),
    
20.   t => {
    
21.    val rowKey = t._1
    
22.    val family:Array[Byte] = t._2(0)._1
    
23.    val qualifier = t._2(0)._2
    
24.    val value = t._2(0)._3
    
25. 
    
26.    val keyFamilyQualifier= new KeyFamilyQualifier(rowKey, family, qualifier)
    
27. 
    
28.    Seq((keyFamilyQualifier, value)).iterator
    
29.   },
    
30.   stagingFolder.getPath,
    
31.   familyHBaseWriterOptions,
    
32.   compactionExclude = false,
    
33.   HConstants.DEFAULT_MAX_FILE_SIZE)
    
34. 
    
35. val load = new LoadIncrementalHFiles(config)
    
36. load.doBulkLoad(new Path(stagingFolder.getPath),
    
37.   conn.getAdmin, table, conn.getRegionLocator(TableName.valueOf(tableName)))

复制代码

1. Example 50. Using thin record bulk load
   
2. val sc = new SparkContext("local", "test")
   
3. val config = new HBaseConfiguration()
   
4. 
   
5. val hbaseContext = new HBaseContext(sc, config)
   
6. 
   
7. val stagingFolder = ...
   
8. val rdd = sc.parallelize(Array(
   
9.       ("1",
   
10.         (Bytes.toBytes(columnFamily1), Bytes.toBytes("a"), Bytes.toBytes("foo1"))),
    
11.       ("3",
    
12.         (Bytes.toBytes(columnFamily1), Bytes.toBytes("b"), Bytes.toBytes("foo2.b"))), ...
    
13. 
    
14. rdd.hbaseBulkLoadThinRows(hbaseContext,
    
15.       TableName.valueOf(tableName),
    
16.       t => {
    
17.         val rowKey = t._1
    
18. 
    
19.         val familyQualifiersValues = new FamiliesQualifiersValues
    
20.         t._2.foreach(f => {
    
21.           val family:Array[Byte] = f._1
    
22.           val qualifier = f._2
    
23.           val value:Array[Byte] = f._3
    
24. 
    
25.           familyQualifiersValues +=(family, qualifier, value)
    
26.         })
    
27.         (new ByteArrayWrapper(Bytes.toBytes(rowKey)), familyQualifiersValues)
    
28.       },
    
29.       stagingFolder.getPath,
    
30.       new java.util.HashMap[Array[Byte], FamilyHFileWriteOptions],
    
31.       compactionExclude = false,
    
32.       20)
    
33. 
    
34. val load = new LoadIncrementalHFiles(config)
    
35. load.doBulkLoad(new Path(stagingFolder.getPath),
    
36.   conn.getAdmin, table, conn.getRegionLocator(TableName.valueOf(tableName)))

复制代码

1. Example 11. Examples
   
2. #Create a namespace
   
3. create_namespace 'my_ns'
   
4. #create my_table in my_ns namespace
   
5. create 'my_ns:my_table', 'fam'
   
6. #drop namespace
   
7. drop_namespace 'my_ns'
   
8. #alter namespace
   
9. alter_namespace 'my_ns', {METHOD => 'set', 'PROPERTY_NAME' => 'PROPERTY_VALUE'}

复制代码

1. Example 12. Examples
   
2. #namespace=foo and table qualifier=bar
   
3. create 'foo:bar', 'fam'
   
4. 
   
5. #namespace=default and table qualifier=bar
   
6. create 'bar', 'fam'

复制代码

1. Example 13. Modify the Maximum Number of Versions for a Column Family
   
2. This example uses HBase Shell to keep a maximum of 5 versions of all columns in column family f1. You could also use HColumnDescriptor.
   
3. 
   
4. hbase> alter ‘t1′, NAME => ‘f1′, VERSIONS => 5

复制代码