Groonga
Loading...
Searching...
No Matches
Classes | Macros | Typedefs | Functions
tokenizer.h File Reference
#include <groonga/plugin.h>
#include <groonga/tokenizer_query_deprecated.h>
Include dependency graph for tokenizer.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes

struct  _grn_tokenizer_token
 

Macros

#define GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8   "\xEF\xBF\xBE"
 
#define GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8_LEN   3
 
#define GRN_TOKENIZER_BEGIN_MARK_UTF8   "\xEF\xBF\xAF"
 
#define GRN_TOKENIZER_BEGIN_MARK_UTF8_LEN   3
 
#define GRN_TOKENIZER_END_MARK_UTF8   "\xEF\xBF\xB0"
 
#define GRN_TOKENIZER_END_MARK_UTF8_LEN   3
 
#define GRN_TOKENIZER_TOKEN_CONTINUE   GRN_TOKEN_CONTINUE
 
#define GRN_TOKENIZER_TOKEN_LAST   GRN_TOKEN_LAST
 
#define GRN_TOKENIZER_TOKEN_OVERLAP   GRN_TOKEN_OVERLAP
 
#define GRN_TOKENIZER_TOKEN_UNMATURED   GRN_TOKEN_UNMATURED
 
#define GRN_TOKENIZER_TOKEN_REACH_END   GRN_TOKEN_REACH_END
 
#define GRN_TOKENIZER_TOKEN_SKIP   GRN_TOKEN_SKIP
 
#define GRN_TOKENIZER_TOKEN_SKIP_WITH_POSITION   GRN_TOKEN_SKIP_WITH_POSITION
 
#define GRN_TOKENIZER_TOKEN_FORCE_PREFIX   GRN_TOKEN_FORCE_PREFIX
 
#define GRN_TOKENIZER_CONTINUE   GRN_TOKENIZER_TOKEN_CONTINUE
 
#define GRN_TOKENIZER_LAST   GRN_TOKENIZER_TOKEN_LAST
 

Typedefs

typedef struct grn_tokenizer_build_data grn_tokenizer_build_data
 This is an opaque data to pass build data to a tokenizer from Groonga.
 
typedef struct _grn_tokenizer_token grn_tokenizer_token
 
typedef grn_token_status grn_tokenizer_status
 
typedef grn_rc grn_tokenizer_build_func(grn_ctx *ctx, grn_tokenizer_build_data *data)
 A function to build an index column by this tokenizer.
 
typedef void * grn_tokenizer_init_func(grn_ctx *ctx, grn_tokenizer_query *query)
 
typedef void grn_tokenizer_next_func(grn_ctx *ctx, grn_tokenizer_query *query, grn_token *token, void *user_data)
 
typedef void grn_tokenizer_fin_func(grn_ctx *ctx, void *user_data)
 

Functions

int grn_tokenizer_charlen (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
 
int grn_tokenizer_isspace (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
 
bool grn_tokenizer_is_tokenized_delimiter (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
 
GRN_PLUGIN_EXPORT bool grn_tokenizer_have_tokenized_delimiter (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
 
GRN_PLUGIN_EXPORT grn_tokenizer_query * grn_tokenizer_query_open (grn_ctx *ctx, int num_args, grn_obj **args, uint32_t normalize_flags)
 
grn_tokenizer_query * grn_tokenizer_query_create (grn_ctx *ctx, int num_args, grn_obj **args)
 
GRN_PLUGIN_EXPORT void grn_tokenizer_query_close (grn_ctx *ctx, grn_tokenizer_query *query)
 
void grn_tokenizer_query_destroy (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_query_set_normalize_flags (grn_ctx *ctx, grn_tokenizer_query *query, uint32_t flags)
 
GRN_PLUGIN_EXPORT uint32_t grn_tokenizer_query_get_normalize_flags (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_normalized_string (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT const char * grn_tokenizer_query_get_raw_string (grn_ctx *ctx, grn_tokenizer_query *query, size_t *length)
 
GRN_PLUGIN_EXPORT const char * grn_tokenizer_query_get_data (grn_ctx *ctx, grn_tokenizer_query *query, size_t *size, grn_id *domain)
 
GRN_PLUGIN_EXPORT grn_encoding grn_tokenizer_query_get_encoding (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT uint32_t grn_tokenizer_query_get_flags (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT bool grn_tokenizer_query_have_tokenized_delimiter (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT grn_tokenize_mode grn_tokenizer_query_get_mode (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_lexicon (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT unsigned int grn_tokenizer_query_get_token_filter_index (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_source_column (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT grn_id grn_tokenizer_query_get_source_id (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_index_column (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_options (grn_ctx *ctx, grn_tokenizer_query *query)
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_build_data_get_source_table (grn_ctx *ctx, grn_tokenizer_build_data *data)
 Return the build target table.
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_build_data_get_source_columns (grn_ctx *ctx, grn_tokenizer_build_data *data)
 Return the build target columns as GRN_PVECTOR.
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_build_data_get_lexicon (grn_ctx *ctx, grn_tokenizer_build_data *data)
 Return the lexicon for this tokenizer.
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_build_data_get_index_column (grn_ctx *ctx, grn_tokenizer_build_data *data)
 Return the index column for this tokenizer.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_processed_n_records (grn_ctx *ctx, grn_tokenizer_build_data *data, uint32_t n_records)
 A tokenizer may call this when it processed one or more records. This is only for progress report.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_vectorize (grn_ctx *ctx, grn_tokenizer_build_data *data, uint32_t n_total_records)
 A tokenizer must call this when it starts vectorizing. This is only for progress report.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_cluster (grn_ctx *ctx, grn_tokenizer_build_data *data, uint32_t n_total_records)
 A tokenizer must call this when it starts clustering. This is only for progress report.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_load (grn_ctx *ctx, grn_tokenizer_build_data *data, uint32_t n_total_records)
 A tokenizer must call this when it starts loading. This is only for progress report.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_record (grn_ctx *ctx, grn_tokenizer_build_data *data, grn_id rid)
 A tokenizer must call this when it starts processing a new record.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_section (grn_ctx *ctx, grn_tokenizer_build_data *data, uint32_t sid)
 A tokenizer must call this when it starts processing a new section.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_append_tokens (grn_ctx *ctx, grn_tokenizer_build_data *data, grn_obj *tokens)
 A tokenizer must call this when it appends tokens to the current record and section.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_finish_section (grn_ctx *ctx, grn_tokenizer_build_data *data)
 A tokenizer must call this when it finishes processing a section.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_finish_record (grn_ctx *ctx, grn_tokenizer_build_data *data)
 A tokenizer must call this when it finishes processing a record.
 
GRN_PLUGIN_EXPORT void grn_tokenizer_token_init (grn_ctx *ctx, grn_tokenizer_token *token)
 
GRN_PLUGIN_EXPORT void grn_tokenizer_token_fin (grn_ctx *ctx, grn_tokenizer_token *token)
 
GRN_PLUGIN_EXPORT void grn_tokenizer_token_push (grn_ctx *ctx, grn_tokenizer_token *token, const char *str_ptr, unsigned int str_length, grn_token_status status)
 
GRN_PLUGIN_EXPORT const char * grn_tokenizer_tokenized_delimiter_next (grn_ctx *ctx, grn_tokenizer_token *token, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
 
GRN_PLUGIN_EXPORT const char * grn_tokenizer_next_by_tokenized_delimiter (grn_ctx *ctx, grn_token *token, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_register (grn_ctx *ctx, const char *plugin_name_ptr, unsigned int plugin_name_length, grn_proc_func *init, grn_proc_func *next, grn_proc_func *fin)
 
GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_create (grn_ctx *ctx, const char *name, int name_length)
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_set_build_func (grn_ctx *ctx, grn_obj *tokenizer, grn_tokenizer_build_func *build)
 Set build function for this tokenizer.
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_set_init_func (grn_ctx *ctx, grn_obj *tokenizer, grn_tokenizer_init_func *init)
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_set_next_func (grn_ctx *ctx, grn_obj *tokenizer, grn_tokenizer_next_func *next)
 
GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_set_fin_func (grn_ctx *ctx, grn_obj *tokenizer, grn_tokenizer_fin_func *fin)
 

Macro Definition Documentation

◆ GRN_TOKENIZER_BEGIN_MARK_UTF8

#define GRN_TOKENIZER_BEGIN_MARK_UTF8   "\xEF\xBF\xAF"

◆ GRN_TOKENIZER_BEGIN_MARK_UTF8_LEN

#define GRN_TOKENIZER_BEGIN_MARK_UTF8_LEN   3

◆ GRN_TOKENIZER_CONTINUE

#define GRN_TOKENIZER_CONTINUE   GRN_TOKENIZER_TOKEN_CONTINUE

◆ GRN_TOKENIZER_END_MARK_UTF8

#define GRN_TOKENIZER_END_MARK_UTF8   "\xEF\xBF\xB0"

◆ GRN_TOKENIZER_END_MARK_UTF8_LEN

#define GRN_TOKENIZER_END_MARK_UTF8_LEN   3

◆ GRN_TOKENIZER_LAST

#define GRN_TOKENIZER_LAST   GRN_TOKENIZER_TOKEN_LAST

◆ GRN_TOKENIZER_TOKEN_CONTINUE

#define GRN_TOKENIZER_TOKEN_CONTINUE   GRN_TOKEN_CONTINUE

◆ GRN_TOKENIZER_TOKEN_FORCE_PREFIX

#define GRN_TOKENIZER_TOKEN_FORCE_PREFIX   GRN_TOKEN_FORCE_PREFIX

◆ GRN_TOKENIZER_TOKEN_LAST

#define GRN_TOKENIZER_TOKEN_LAST   GRN_TOKEN_LAST

◆ GRN_TOKENIZER_TOKEN_OVERLAP

#define GRN_TOKENIZER_TOKEN_OVERLAP   GRN_TOKEN_OVERLAP

◆ GRN_TOKENIZER_TOKEN_REACH_END

#define GRN_TOKENIZER_TOKEN_REACH_END   GRN_TOKEN_REACH_END

◆ GRN_TOKENIZER_TOKEN_SKIP

#define GRN_TOKENIZER_TOKEN_SKIP   GRN_TOKEN_SKIP

◆ GRN_TOKENIZER_TOKEN_SKIP_WITH_POSITION

#define GRN_TOKENIZER_TOKEN_SKIP_WITH_POSITION   GRN_TOKEN_SKIP_WITH_POSITION

◆ GRN_TOKENIZER_TOKEN_UNMATURED

#define GRN_TOKENIZER_TOKEN_UNMATURED   GRN_TOKEN_UNMATURED

◆ GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8

#define GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8   "\xEF\xBF\xBE"

◆ GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8_LEN

#define GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8_LEN   3

Typedef Documentation

◆ grn_tokenizer_build_data

typedef struct grn_tokenizer_build_data grn_tokenizer_build_data

This is an opaque data to pass build data to a tokenizer from Groonga.

This is used for grn_tokenizer_build_func.

See also
grn_tokenizer_set_build_func
Since
15.1.8

◆ grn_tokenizer_build_func

typedef grn_rc grn_tokenizer_build_func(grn_ctx *ctx, grn_tokenizer_build_data *data)

A function to build an index column by this tokenizer.

Parameters
ctxThe context object.
dataData to be built.
Returns
GRN_SUCCESS on success, the appropriate grn_rc on error. ctx->rc must be updated too by GRN_PLUGIN_ERROR on error.
See also
grn_tokenizer_set_build_func.
Since
15.1.8

◆ grn_tokenizer_fin_func

typedef void grn_tokenizer_fin_func(grn_ctx *ctx, void *user_data)

◆ grn_tokenizer_init_func

typedef void * grn_tokenizer_init_func(grn_ctx *ctx, grn_tokenizer_query *query)

◆ grn_tokenizer_next_func

typedef void grn_tokenizer_next_func(grn_ctx *ctx, grn_tokenizer_query *query, grn_token *token, void *user_data)

◆ grn_tokenizer_status

typedef grn_token_status grn_tokenizer_status

◆ grn_tokenizer_token

typedef struct _grn_tokenizer_token grn_tokenizer_token

Function Documentation

◆ grn_tokenizer_build_data_append_tokens()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_append_tokens ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data,
grn_obj *  tokens 
)

A tokenizer must call this when it appends tokens to the current record and section.

This can't be called multiple times for a pair of record and section.

Since
15.1.8

◆ grn_tokenizer_build_data_finish_record()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_finish_record ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data 
)

A tokenizer must call this when it finishes processing a record.

Since
15.1.8

◆ grn_tokenizer_build_data_finish_section()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_finish_section ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data 
)

A tokenizer must call this when it finishes processing a section.

Since
15.1.8

◆ grn_tokenizer_build_data_get_index_column()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_build_data_get_index_column ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data 
)

Return the index column for this tokenizer.

This is owned by grn_tokenizer_build_data. You must not free it.

Since
15.1.8

◆ grn_tokenizer_build_data_get_lexicon()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_build_data_get_lexicon ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data 
)

Return the lexicon for this tokenizer.

This is owned by grn_tokenizer_build_data. You must not free it.

Since
15.1.8

◆ grn_tokenizer_build_data_get_source_columns()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_build_data_get_source_columns ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data 
)

Return the build target columns as GRN_PVECTOR.

This is owned by grn_tokenizer_build_data. You must not free it.

Since
15.1.8

◆ grn_tokenizer_build_data_get_source_table()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_build_data_get_source_table ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data 
)

Return the build target table.

This is owned by grn_tokenizer_build_data. You must not free it.

Since
15.1.8

◆ grn_tokenizer_build_data_processed_n_records()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_processed_n_records ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data,
uint32_t  n_records 
)

A tokenizer may call this when it processed one or more records. This is only for progress report.

Parameters
ctxThe context object.
dataThe tokenizer build data.
n_recordsThe number of processed records.
Since
15.2.1

◆ grn_tokenizer_build_data_start_cluster()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_cluster ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data,
uint32_t  n_total_records 
)

A tokenizer must call this when it starts clustering. This is only for progress report.

Parameters
ctxThe context object.
dataThe tokenizer build data.
n_total_recordsThe number of records to be clustered.
Since
15.2.1

◆ grn_tokenizer_build_data_start_load()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_load ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data,
uint32_t  n_total_records 
)

A tokenizer must call this when it starts loading. This is only for progress report.

Parameters
ctxThe context object.
dataThe tokenizer build data.
n_total_recordsThe number of records to be loaded.
Since
15.2.1

◆ grn_tokenizer_build_data_start_record()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_record ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data,
grn_id  rid 
)

A tokenizer must call this when it starts processing a new record.

Since
15.1.8

◆ grn_tokenizer_build_data_start_section()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_section ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data,
uint32_t  sid 
)

A tokenizer must call this when it starts processing a new section.

Since
15.1.8

◆ grn_tokenizer_build_data_start_vectorize()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_build_data_start_vectorize ( grn_ctx *  ctx,
grn_tokenizer_build_data *  data,
uint32_t  n_total_records 
)

A tokenizer must call this when it starts vectorizing. This is only for progress report.

Parameters
ctxThe context object.
dataThe tokenizer build data.
n_total_recordsThe number of records to be vectorized.
Since
15.2.1

◆ grn_tokenizer_charlen()

int grn_tokenizer_charlen ( grn_ctx *  ctx,
const char *  str_ptr,
unsigned int  str_length,
grn_encoding  encoding 
)

◆ grn_tokenizer_create()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_create ( grn_ctx *  ctx,
const char *  name,
int  name_length 
)

◆ grn_tokenizer_have_tokenized_delimiter()

GRN_PLUGIN_EXPORT bool grn_tokenizer_have_tokenized_delimiter ( grn_ctx *  ctx,
const char *  str_ptr,
unsigned int  str_length,
grn_encoding  encoding 
)

◆ grn_tokenizer_is_tokenized_delimiter()

bool grn_tokenizer_is_tokenized_delimiter ( grn_ctx *  ctx,
const char *  str_ptr,
unsigned int  str_length,
grn_encoding  encoding 
)

◆ grn_tokenizer_isspace()

int grn_tokenizer_isspace ( grn_ctx *  ctx,
const char *  str_ptr,
unsigned int  str_length,
grn_encoding  encoding 
)

◆ grn_tokenizer_next_by_tokenized_delimiter()

GRN_PLUGIN_EXPORT const char * grn_tokenizer_next_by_tokenized_delimiter ( grn_ctx *  ctx,
grn_token *  token,
const char *  str_ptr,
unsigned int  str_length,
grn_encoding  encoding 
)

◆ grn_tokenizer_query_close()

GRN_PLUGIN_EXPORT void grn_tokenizer_query_close ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_create()

grn_tokenizer_query * grn_tokenizer_query_create ( grn_ctx *  ctx,
int  num_args,
grn_obj **  args 
)

◆ grn_tokenizer_query_destroy()

void grn_tokenizer_query_destroy ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_data()

GRN_PLUGIN_EXPORT const char * grn_tokenizer_query_get_data ( grn_ctx *  ctx,
grn_tokenizer_query *  query,
size_t *  size,
grn_id *  domain 
)

◆ grn_tokenizer_query_get_encoding()

GRN_PLUGIN_EXPORT grn_encoding grn_tokenizer_query_get_encoding ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_flags()

GRN_PLUGIN_EXPORT uint32_t grn_tokenizer_query_get_flags ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_index_column()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_index_column ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_lexicon()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_lexicon ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_mode()

GRN_PLUGIN_EXPORT grn_tokenize_mode grn_tokenizer_query_get_mode ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_normalize_flags()

GRN_PLUGIN_EXPORT uint32_t grn_tokenizer_query_get_normalize_flags ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_normalized_string()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_normalized_string ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_options()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_options ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_raw_string()

GRN_PLUGIN_EXPORT const char * grn_tokenizer_query_get_raw_string ( grn_ctx *  ctx,
grn_tokenizer_query *  query,
size_t *  length 
)

◆ grn_tokenizer_query_get_source_column()

GRN_PLUGIN_EXPORT grn_obj * grn_tokenizer_query_get_source_column ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_source_id()

GRN_PLUGIN_EXPORT grn_id grn_tokenizer_query_get_source_id ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_get_token_filter_index()

GRN_PLUGIN_EXPORT unsigned int grn_tokenizer_query_get_token_filter_index ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_have_tokenized_delimiter()

GRN_PLUGIN_EXPORT bool grn_tokenizer_query_have_tokenized_delimiter ( grn_ctx *  ctx,
grn_tokenizer_query *  query 
)

◆ grn_tokenizer_query_open()

GRN_PLUGIN_EXPORT grn_tokenizer_query * grn_tokenizer_query_open ( grn_ctx *  ctx,
int  num_args,
grn_obj **  args,
uint32_t  normalize_flags 
)

◆ grn_tokenizer_query_set_normalize_flags()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_query_set_normalize_flags ( grn_ctx *  ctx,
grn_tokenizer_query *  query,
uint32_t  flags 
)

◆ grn_tokenizer_register()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_register ( grn_ctx *  ctx,
const char *  plugin_name_ptr,
unsigned int  plugin_name_length,
grn_proc_func *  init,
grn_proc_func *  next,
grn_proc_func *  fin 
)

◆ grn_tokenizer_set_build_func()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_set_build_func ( grn_ctx *  ctx,
grn_obj *  tokenizer,
grn_tokenizer_build_func *  build 
)

Set build function for this tokenizer.

The given build is called only once when an index column that uses this tokenizer is created.

See also
grn_tokenizer_build_func
Since
15.1.8

◆ grn_tokenizer_set_fin_func()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_set_fin_func ( grn_ctx *  ctx,
grn_obj *  tokenizer,
grn_tokenizer_fin_func *  fin 
)

◆ grn_tokenizer_set_init_func()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_set_init_func ( grn_ctx *  ctx,
grn_obj *  tokenizer,
grn_tokenizer_init_func *  init 
)

◆ grn_tokenizer_set_next_func()

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_set_next_func ( grn_ctx *  ctx,
grn_obj *  tokenizer,
grn_tokenizer_next_func *  next 
)

◆ grn_tokenizer_token_fin()

GRN_PLUGIN_EXPORT void grn_tokenizer_token_fin ( grn_ctx *  ctx,
grn_tokenizer_token *  token 
)

◆ grn_tokenizer_token_init()

GRN_PLUGIN_EXPORT void grn_tokenizer_token_init ( grn_ctx *  ctx,
grn_tokenizer_token *  token 
)

◆ grn_tokenizer_token_push()

GRN_PLUGIN_EXPORT void grn_tokenizer_token_push ( grn_ctx *  ctx,
grn_tokenizer_token *  token,
const char *  str_ptr,
unsigned int  str_length,
grn_token_status  status 
)

◆ grn_tokenizer_tokenized_delimiter_next()

GRN_PLUGIN_EXPORT const char * grn_tokenizer_tokenized_delimiter_next ( grn_ctx *  ctx,
grn_tokenizer_token *  token,
const char *  str_ptr,
unsigned int  str_length,
grn_encoding  encoding 
)