org.apache.accumulo.core.client.mapreduce

Class AccumuloOutputFormat

java.lang.Object

org.apache.hadoop.mapreduce.OutputFormat<org.apache.hadoop.io.Text,Mutation>

org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat

public class AccumuloOutputFormat
extends org.apache.hadoop.mapreduce.OutputFormat<org.apache.hadoop.io.Text,Mutation>

This class allows MapReduce jobs to use Accumulo as the sink for data. This OutputFormat accepts keys and values of type Text (for a table name) and Mutation from the Map and Reduce functions. The user must specify the following via static configurator methods:

• setConnectorInfo(Job, String, AuthenticationToken)
• setZooKeeperInstance(Job, String, String) OR setMockInstance(Job, String)

Other static methods are optional.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
protected static class	AccumuloOutputFormat.AccumuloRecordWriter A base class to be used to create RecordWriter instances that write to Accumulo.

Field Summary

Fields

Modifier and Type	Field and Description
protected static org.apache.log4j.Logger	log

Constructor Summary

Constructors

Constructor and Description
AccumuloOutputFormat()

Method Summary

Methods

Modifier and Type	Method and Description
protected static boolean	canCreateTables(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use canCreateTables(JobContext) instead.
protected static Boolean	canCreateTables(org.apache.hadoop.mapreduce.JobContext context) Determines whether tables are permitted to be created as needed.
void	checkOutputSpecs(org.apache.hadoop.mapreduce.JobContext job)
protected static BatchWriterConfig	getBatchWriterOptions(org.apache.hadoop.mapreduce.JobContext context) Gets the BatchWriterConfig settings.
protected static String	getDefaultTableName(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use getDefaultTableName(JobContext) instead.
protected static String	getDefaultTableName(org.apache.hadoop.mapreduce.JobContext context) Gets the default table name from the configuration.
protected static Instance	getInstance(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use getInstance(JobContext) instead.
protected static Instance	getInstance(org.apache.hadoop.mapreduce.JobContext context) Initializes an Accumulo Instance based on the configuration.
protected static org.apache.log4j.Level	getLogLevel(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use getLogLevel(JobContext) instead.
protected static org.apache.log4j.Level	getLogLevel(org.apache.hadoop.mapreduce.JobContext context) Gets the log level from this configuration.
protected static int	getMaxLatency(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use getBatchWriterOptions(JobContext) instead.
protected static long	getMaxMutationBufferSize(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use getBatchWriterOptions(JobContext) instead.
protected static int	getMaxWriteThreads(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use getBatchWriterOptions(JobContext) instead.
org.apache.hadoop.mapreduce.OutputCommitter	getOutputCommitter(org.apache.hadoop.mapreduce.TaskAttemptContext context)
protected static String	getPrincipal(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use getToken(JobContext) instead.
protected static String	getPrincipal(org.apache.hadoop.mapreduce.JobContext context) Gets the user name from the configuration.
org.apache.hadoop.mapreduce.RecordWriter<org.apache.hadoop.io.Text,Mutation>	getRecordWriter(org.apache.hadoop.mapreduce.TaskAttemptContext attempt)
protected static boolean	getSimulationMode(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use getSimulationMode(JobContext) instead.
protected static Boolean	getSimulationMode(org.apache.hadoop.mapreduce.JobContext context) Determines whether this feature is enabled.
protected static byte[]	getToken(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use getToken(JobContext) instead.
protected static byte[]	getToken(org.apache.hadoop.mapreduce.JobContext context) Gets the password from the configuration.
protected static String	getTokenClass(org.apache.hadoop.mapreduce.JobContext context) Gets the serialized token class name from the configuration.
protected static Boolean	isConnectorInfoSet(org.apache.hadoop.mapreduce.JobContext context) Determines if the connector has been configured.
static void	setBatchWriterOptions(org.apache.hadoop.mapreduce.Job job, BatchWriterConfig bwConfig) Sets the configuration for for the job's BatchWriter instances.
static void	setConnectorInfo(org.apache.hadoop.mapreduce.Job job, String principal, AuthenticationToken token) Sets the connector information needed to communicate with Accumulo in this job.
static void	setCreateTables(org.apache.hadoop.mapreduce.Job job, boolean enableFeature) Sets the directive to create new tables, as necessary.
static void	setDefaultTableName(org.apache.hadoop.mapreduce.Job job, String tableName) Sets the default table name to use if one emits a null in place of a table name for a given mutation.
static void	setLogLevel(org.apache.hadoop.conf.Configuration conf, org.apache.log4j.Level level) Deprecated. since 1.5.0; Use setLogLevel(Job, Level) instead.
static void	setLogLevel(org.apache.hadoop.mapreduce.Job job, org.apache.log4j.Level level) Sets the log level for this job.
static void	setMaxLatency(org.apache.hadoop.conf.Configuration conf, int numberOfMilliseconds) Deprecated. since 1.5.0; Use setBatchWriterOptions(Job, BatchWriterConfig) instead.
static void	setMaxMutationBufferSize(org.apache.hadoop.conf.Configuration conf, long numberOfBytes) Deprecated. since 1.5.0; Use setBatchWriterOptions(Job, BatchWriterConfig) instead.
static void	setMaxWriteThreads(org.apache.hadoop.conf.Configuration conf, int numberOfThreads) Deprecated. since 1.5.0; Use setBatchWriterOptions(Job, BatchWriterConfig) instead.
static void	setMockInstance(org.apache.hadoop.conf.Configuration conf, String instanceName) Deprecated. since 1.5.0; Use setMockInstance(Job, String) instead.
static void	setMockInstance(org.apache.hadoop.mapreduce.Job job, String instanceName) Configures a MockInstance for this job.
static void	setOutputInfo(org.apache.hadoop.conf.Configuration conf, String user, byte[] passwd, boolean createTables, String defaultTable) Deprecated. since 1.5.0; Use setConnectorInfo(Job, String, AuthenticationToken), setCreateTables(Job, boolean), and setDefaultTableName(Job, String) instead.
static void	setSimulationMode(org.apache.hadoop.conf.Configuration conf) Deprecated. since 1.5.0; Use setSimulationMode(Job, boolean) instead.
static void	setSimulationMode(org.apache.hadoop.mapreduce.Job job, boolean enableFeature) Sets the directive to use simulation mode for this job.
static void	setZooKeeperInstance(org.apache.hadoop.conf.Configuration conf, String instanceName, String zooKeepers) Deprecated. since 1.5.0; Use setZooKeeperInstance(Job, String, String) instead.
static void	setZooKeeperInstance(org.apache.hadoop.mapreduce.Job job, String instanceName, String zooKeepers) Configures a ZooKeeperInstance for this job.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

log

protected static final org.apache.log4j.Logger log

Constructor Detail

AccumuloOutputFormat

public AccumuloOutputFormat()

Method Detail

setConnectorInfo

public static void setConnectorInfo(org.apache.hadoop.mapreduce.Job job,
                    String principal,
                    AuthenticationToken token)
                             throws AccumuloSecurityException

Sets the connector information needed to communicate with Accumulo in this job.

WARNING: The serialized token is stored in the configuration and shared with all MapReduce tasks. It is BASE64 encoded to provide a charset safe conversion to a string, and is not intended to be secure.

Parameters:

job - the Hadoop job instance to be configured

principal - a valid Accumulo user name (user must have Table.CREATE permission if setCreateTables(Job, boolean) is set to true)

token - the user's password

Throws:

AccumuloSecurityException

Since:

1.5.0

isConnectorInfoSet

protected static Boolean isConnectorInfoSet(org.apache.hadoop.mapreduce.JobContext context)

Determines if the connector has been configured.

Parameters:

context - the Hadoop context for the configured job

Returns:

true if the connector has been configured, false otherwise

Since:

1.5.0

See Also:

setConnectorInfo(Job, String, AuthenticationToken)

getPrincipal

protected static String getPrincipal(org.apache.hadoop.mapreduce.JobContext context)

Gets the user name from the configuration.

Parameters:

context - the Hadoop context for the configured job

Returns:

the user name

Since:

1.5.0

See Also:

setConnectorInfo(Job, String, AuthenticationToken)

getTokenClass

protected static String getTokenClass(org.apache.hadoop.mapreduce.JobContext context)

Gets the serialized token class name from the configuration.

Parameters:

context - the Hadoop context for the configured job

Returns:

the user name

Since:

1.5.0

See Also:

setConnectorInfo(Job, String, AuthenticationToken)

getToken

protected static byte[] getToken(org.apache.hadoop.mapreduce.JobContext context)

Gets the password from the configuration. WARNING: The password is stored in the Configuration and shared with all MapReduce tasks; It is BASE64 encoded to provide a charset safe conversion to a string, and is not intended to be secure.

Parameters:

context - the Hadoop context for the configured job

Returns:

the decoded user password

Since:

1.5.0

See Also:

setConnectorInfo(Job, String, AuthenticationToken)

setZooKeeperInstance

public static void setZooKeeperInstance(org.apache.hadoop.mapreduce.Job job,
                        String instanceName,
                        String zooKeepers)

Configures a ZooKeeperInstance for this job.

Parameters:

job - the Hadoop job instance to be configured

instanceName - the Accumulo instance name

zooKeepers - a comma-separated list of zookeeper servers

Since:

1.5.0

setMockInstance

public static void setMockInstance(org.apache.hadoop.mapreduce.Job job,
                   String instanceName)

Configures a MockInstance for this job.

Parameters:

job - the Hadoop job instance to be configured

instanceName - the Accumulo instance name

Since:

1.5.0

getInstance

protected static Instance getInstance(org.apache.hadoop.mapreduce.JobContext context)

Initializes an Accumulo Instance based on the configuration.

Parameters:

context - the Hadoop context for the configured job

Returns:

an Accumulo instance

Since:

1.5.0

See Also:

setZooKeeperInstance(Job, String, String), setMockInstance(Job, String)

setLogLevel

public static void setLogLevel(org.apache.hadoop.mapreduce.Job job,
               org.apache.log4j.Level level)

Sets the log level for this job.

Parameters:

job - the Hadoop job instance to be configured

level - the logging level

Since:

1.5.0

getLogLevel

protected static org.apache.log4j.Level getLogLevel(org.apache.hadoop.mapreduce.JobContext context)

Gets the log level from this configuration.

Parameters:

context - the Hadoop context for the configured job

Returns:

the log level

Since:

1.5.0

See Also:

setLogLevel(Job, Level)

setDefaultTableName

public static void setDefaultTableName(org.apache.hadoop.mapreduce.Job job,
                       String tableName)

Sets the default table name to use if one emits a null in place of a table name for a given mutation. Table names can only be alpha-numeric and underscores.

Parameters:

job - the Hadoop job instance to be configured

tableName - the table to use when the tablename is null in the write call

Since:

1.5.0

getDefaultTableName

protected static String getDefaultTableName(org.apache.hadoop.mapreduce.JobContext context)

Gets the default table name from the configuration.

Parameters:

context - the Hadoop context for the configured job

Returns:

the default table name

Since:

1.5.0

See Also:

setDefaultTableName(Job, String)

setBatchWriterOptions

public static void setBatchWriterOptions(org.apache.hadoop.mapreduce.Job job,
                         BatchWriterConfig bwConfig)

Sets the configuration for for the job's BatchWriter instances. If not set, a new BatchWriterConfig, with sensible built-in defaults is used. Setting the configuration multiple times overwrites any previous configuration.

Parameters:

job - the Hadoop job instance to be configured

bwConfig - the configuration for the BatchWriter

Since:

1.5.0

getBatchWriterOptions

protected static BatchWriterConfig getBatchWriterOptions(org.apache.hadoop.mapreduce.JobContext context)

Gets the BatchWriterConfig settings.

Parameters:

context - the Hadoop context for the configured job

Returns:

the configuration object

Since:

1.5.0

See Also:

setBatchWriterOptions(Job, BatchWriterConfig)

setCreateTables

public static void setCreateTables(org.apache.hadoop.mapreduce.Job job,
                   boolean enableFeature)

Sets the directive to create new tables, as necessary. Table names can only be alpha-numeric and underscores.

By default, this feature is disabled.

Parameters:

job - the Hadoop job instance to be configured

enableFeature - the feature is enabled if true, disabled otherwise

Since:

1.5.0

canCreateTables

protected static Boolean canCreateTables(org.apache.hadoop.mapreduce.JobContext context)

Determines whether tables are permitted to be created as needed.

Parameters:

context - the Hadoop context for the configured job

Returns:

true if the feature is disabled, false otherwise

Since:

1.5.0

See Also:

setCreateTables(Job, boolean)

setSimulationMode

public static void setSimulationMode(org.apache.hadoop.mapreduce.Job job,
                     boolean enableFeature)

Sets the directive to use simulation mode for this job. In simulation mode, no output is produced. This is useful for testing.

By default, this feature is disabled.

Parameters:

job - the Hadoop job instance to be configured

enableFeature - the feature is enabled if true, disabled otherwise

Since:

1.5.0

getSimulationMode

protected static Boolean getSimulationMode(org.apache.hadoop.mapreduce.JobContext context)

Determines whether this feature is enabled.

Parameters:

context - the Hadoop context for the configured job

Returns:

true if the feature is enabled, false otherwise

Since:

1.5.0

See Also:

setSimulationMode(Job, boolean)

checkOutputSpecs

public void checkOutputSpecs(org.apache.hadoop.mapreduce.JobContext job)
                      throws IOException

Specified by:

checkOutputSpecs in class org.apache.hadoop.mapreduce.OutputFormat<org.apache.hadoop.io.Text,Mutation>

Throws:

IOException

getOutputCommitter

public org.apache.hadoop.mapreduce.OutputCommitter getOutputCommitter(org.apache.hadoop.mapreduce.TaskAttemptContext context)

Specified by:

getOutputCommitter in class org.apache.hadoop.mapreduce.OutputFormat<org.apache.hadoop.io.Text,Mutation>

getRecordWriter

public org.apache.hadoop.mapreduce.RecordWriter<org.apache.hadoop.io.Text,Mutation> getRecordWriter(org.apache.hadoop.mapreduce.TaskAttemptContext attempt)
                                                                                             throws IOException

Specified by:

getRecordWriter in class org.apache.hadoop.mapreduce.OutputFormat<org.apache.hadoop.io.Text,Mutation>

Throws:

IOException

setOutputInfo

@Deprecated
public static void setOutputInfo(org.apache.hadoop.conf.Configuration conf,
                            String user,
                            byte[] passwd,
                            boolean createTables,
                            String defaultTable)

Deprecated. since 1.5.0; Use setConnectorInfo(Job, String, AuthenticationToken), setCreateTables(Job, boolean), and setDefaultTableName(Job, String) instead.

setZooKeeperInstance

@Deprecated
public static void setZooKeeperInstance(org.apache.hadoop.conf.Configuration conf,
                                   String instanceName,
                                   String zooKeepers)

Deprecated. since 1.5.0; Use setZooKeeperInstance(Job, String, String) instead.

setMockInstance

@Deprecated
public static void setMockInstance(org.apache.hadoop.conf.Configuration conf,
                              String instanceName)

Deprecated. since 1.5.0; Use setMockInstance(Job, String) instead.

setMaxMutationBufferSize

@Deprecated
public static void setMaxMutationBufferSize(org.apache.hadoop.conf.Configuration conf,
                                       long numberOfBytes)

Deprecated. since 1.5.0; Use setBatchWriterOptions(Job, BatchWriterConfig) instead.

setMaxLatency

@Deprecated
public static void setMaxLatency(org.apache.hadoop.conf.Configuration conf,
                            int numberOfMilliseconds)

Deprecated. since 1.5.0; Use setBatchWriterOptions(Job, BatchWriterConfig) instead.

setMaxWriteThreads

@Deprecated
public static void setMaxWriteThreads(org.apache.hadoop.conf.Configuration conf,
                                 int numberOfThreads)

Deprecated. since 1.5.0; Use setBatchWriterOptions(Job, BatchWriterConfig) instead.

setLogLevel

@Deprecated
public static void setLogLevel(org.apache.hadoop.conf.Configuration conf,
                          org.apache.log4j.Level level)

Deprecated. since 1.5.0; Use setLogLevel(Job, Level) instead.

setSimulationMode

@Deprecated
public static void setSimulationMode(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use setSimulationMode(Job, boolean) instead.

getPrincipal

@Deprecated
protected static String getPrincipal(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use getToken(JobContext) instead.

getToken

@Deprecated
protected static byte[] getToken(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use getToken(JobContext) instead.

canCreateTables

@Deprecated
protected static boolean canCreateTables(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use canCreateTables(JobContext) instead.

getDefaultTableName

@Deprecated
protected static String getDefaultTableName(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use getDefaultTableName(JobContext) instead.

getInstance

@Deprecated
protected static Instance getInstance(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use getInstance(JobContext) instead.

getMaxMutationBufferSize

@Deprecated
protected static long getMaxMutationBufferSize(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use getBatchWriterOptions(JobContext) instead.

getMaxLatency

@Deprecated
protected static int getMaxLatency(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use getBatchWriterOptions(JobContext) instead.

getMaxWriteThreads

@Deprecated
protected static int getMaxWriteThreads(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use getBatchWriterOptions(JobContext) instead.

getLogLevel

@Deprecated
protected static org.apache.log4j.Level getLogLevel(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use getLogLevel(JobContext) instead.

getSimulationMode

@Deprecated
protected static boolean getSimulationMode(org.apache.hadoop.conf.Configuration conf)

Deprecated. since 1.5.0; Use getSimulationMode(JobContext) instead.