ikan Uploader :
Directory : /proc/thread-self/root/home/ubuntu/node-v16.18.1/deps/icu-small/source/i18n/
Upload File : |
| Current File : //proc/thread-self/root/home/ubuntu/node-v16.18.1/deps/icu-small/source/i18n/inputext.cpp |
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
**********************************************************************
* Copyright (C) 2005-2016, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
*/
#include "unicode/utypes.h"
#if !UCONFIG_NO_CONVERSION
#include "inputext.h"
#include "cmemory.h"
#include "cstring.h"
#include <string.h>
U_NAMESPACE_BEGIN
#define BUFFER_SIZE 8192
#define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
#define DELETE_ARRAY(array) uprv_free((void *) (array))
InputText::InputText(UErrorCode &status)
: fInputBytes(NEW_ARRAY(uint8_t, BUFFER_SIZE)), // The text to be checked. Markup will have been
// removed if appropriate.
fByteStats(NEW_ARRAY(int16_t, 256)), // byte frequency statistics for the input text.
// Value is percent, not absolute.
fDeclaredEncoding(0),
fRawInput(0),
fRawLength(0)
{
if (fInputBytes == NULL || fByteStats == NULL) {
status = U_MEMORY_ALLOCATION_ERROR;
}
}
InputText::~InputText()
{
DELETE_ARRAY(fDeclaredEncoding);
DELETE_ARRAY(fByteStats);
DELETE_ARRAY(fInputBytes);
}
void InputText::setText(const char *in, int32_t len)
{
fInputLen = 0;
fC1Bytes = FALSE;
fRawInput = (const uint8_t *) in;
fRawLength = len == -1? (int32_t)uprv_strlen(in) : len;
}
void InputText::setDeclaredEncoding(const char* encoding, int32_t len)
{
if(encoding) {
if (len == -1) {
len = (int32_t)uprv_strlen(encoding);
}
len += 1; // to make place for the \0 at the end.
uprv_free(fDeclaredEncoding);
fDeclaredEncoding = NEW_ARRAY(char, len);
uprv_strncpy(fDeclaredEncoding, encoding, len);
}
}
UBool InputText::isSet() const
{
return fRawInput != NULL;
}
/**
* MungeInput - after getting a set of raw input data to be analyzed, preprocess
* it by removing what appears to be html markup.
*
* @internal
*/
void InputText::MungeInput(UBool fStripTags) {
int srci = 0;
int dsti = 0;
uint8_t b;
bool inMarkup = FALSE;
int32_t openTags = 0;
int32_t badTags = 0;
//
// html / xml markup stripping.
// quick and dirty, not 100% accurate, but hopefully good enough, statistically.
// discard everything within < brackets >
// Count how many total '<' and illegal (nested) '<' occur, so we can make some
// guess as to whether the input was actually marked up at all.
// TODO: Think about how this interacts with EBCDIC charsets that are detected.
if (fStripTags) {
for (srci = 0; srci < fRawLength && dsti < BUFFER_SIZE; srci += 1) {
b = fRawInput[srci];
if (b == (uint8_t)0x3C) { /* Check for the ASCII '<' */
if (inMarkup) {
badTags += 1;
}
inMarkup = TRUE;
openTags += 1;
}
if (! inMarkup) {
fInputBytes[dsti++] = b;
}
if (b == (uint8_t)0x3E) { /* Check for the ASCII '>' */
inMarkup = FALSE;
}
}
fInputLen = dsti;
}
//
// If it looks like this input wasn't marked up, or if it looks like it's
// essentially nothing but markup abandon the markup stripping.
// Detection will have to work on the unstripped input.
//
if (openTags<5 || openTags/5 < badTags ||
(fInputLen < 100 && fRawLength>600))
{
int32_t limit = fRawLength;
if (limit > BUFFER_SIZE) {
limit = BUFFER_SIZE;
}
for (srci=0; srci<limit; srci++) {
fInputBytes[srci] = fRawInput[srci];
}
fInputLen = srci;
}
//
// Tally up the byte occurrence statistics.
// These are available for use by the various detectors.
//
uprv_memset(fByteStats, 0, (sizeof fByteStats[0]) * 256);
for (srci = 0; srci < fInputLen; srci += 1) {
fByteStats[fInputBytes[srci]] += 1;
}
for (int32_t i = 0x80; i <= 0x9F; i += 1) {
if (fByteStats[i] != 0) {
fC1Bytes = TRUE;
break;
}
}
}
U_NAMESPACE_END
#endif
Kontol Shell Bypass