internal/cld2_do_score.cc - external/cld2 - Git at Google

 // Copyright 2013 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 // Little program to read lines of sample text, calculate score per 1024 bytes
 // per language-script4 combination
 // Possible input file /export/hda3/cld/pre2010/b0_samp_prune_20100722.utf8

 #include <stdio.h>
 #include <string.h>
 #include <string>

 #include "compact_lang_det_impl.h"
 #include "lang_script.h"

 using namespace std;
 using namespace CLD2;

 double bytes[NUM_LANGUAGES][4];
 double scores[NUM_LANGUAGES][4];


 // Return score per 1024 bytes for top language
 Language ScoreOneLine(const char* buffer, int buffer_length,
                     int* bytes, double* score_per_1kb) {
   bool is_plain_text = true;
   const CLDHints* cld_hints = NULL;
   bool allow_extended_lang = true;
   int flags = 0;
   Language plus_one = UNKNOWN_LANGUAGE;
   Language language3[3];
   int percent3[3];
   double normalized_score3[3];
   ResultChunkVector* resultchunkvector = NULL;
   int text_bytes;
   bool is_reliable;
   Language summary_lang;

   summary_lang = DetectLanguageSummaryV2(
                         buffer,
                         buffer_length,
                         is_plain_text,
                         cld_hints,
                         allow_extended_lang,
                         flags,
                         plus_one,
                         language3,
                         percent3,
                         normalized_score3,
                         resultchunkvector,
                         &text_bytes,
                         &is_reliable);
   *bytes = text_bytes;
   *score_per_1kb = normalized_score3[0];
   return language3[0];
 }

 #define LF 0x0a
 #define CR 0x0d
 const int kMaxBuffer = 5 * 1024;

 bool ReadLine(FILE* infile, char* buffer, size_t maxlen) {
   char* p = fgets(buffer, maxlen, infile);
   if (p == NULL) {
     return false;
   }
   int len = strlen(buffer);

   // trim CR LF
   if (buffer[len-1] == LF) {buffer[--len] = '\0';}
   if (buffer[len-1] == CR) {buffer[--len] = '\0';}
   return true;
 }

 bool IsComment(const char* buffer) {
   int len = strlen(buffer);
   if (len == 0) {return true;}
   if (buffer[0] == '#') {return true;}
   if (buffer[0] == ' ') {return true;}    // Any leading space is comment
   return false;
 }

 // Skips over xxxxx_ where _ is one or more spaces/tabs
 // Returns string::npos if no more fields
 int SkipOneField(const string& src, int pos) {
   if (pos == string::npos) {return pos;}

   int lpos = pos;
   lpos = src.find_first_of(" \t", lpos);
   if (lpos == string::npos) {return lpos;}
   lpos = src.find_first_not_of(" \t", lpos);
   if (lpos == string::npos) {return lpos;}
   return lpos;
 }

 // Return language and script from parsed line or defaults
 void GetLangScript(const string& src,
                    Language default_lang, ULScript default_lscript,
                    Language* target_lang, ULScript* target_lscript,
                    string* tld) {
   *target_lang = default_lang;
   *target_lscript = default_lscript;
   *tld = "";
   int pos = 0;
   int pos2 = 0;
   if (src.substr(0,7) == "SAMPLE ") {
     // SAMPLE ll-Ssss
     pos = SkipOneField(src, pos);
   } else if (src.substr(0,5) == "SAMP ") {
     // SAMP ll-Ssss /tld2.tld/
     pos = SkipOneField(src, pos);
     pos2 = SkipOneField(src, pos);
   } else if (src.substr(0,5) == "Samp ") {
     // Samp ll-Ssss /tld2.tld/
     pos = SkipOneField(src, pos);
     pos2 = SkipOneField(src, pos);
   }
   if (pos == 0) {return;}
   if (pos == string::npos) {return;}

   // Pos is at the first letter of language-script combination
   int end = src.find_first_of(" \t", pos);    // find end of lang-script
   if (end == string::npos) {return;}
   *target_lang = GetLanguageFromName(src.substr(pos, end - pos).c_str());
   *target_lscript = GetULScriptFromName(src.substr(pos, end - pos).c_str());

   // Pos2 is 0 or at the first letter of the tld string
   if (pos2 == 0) {return;}
   if (pos2 == string::npos) {return;}
   end = src.find_first_of(" \t", pos2);
   if (end == string::npos) {return;}
   *tld = src.substr(pos2, end - pos2);
 }

 // Return position of start of text
 int GetTextBeginPos(const string& src) {
   int pos = 0;
   if (src.size() < 8) {return pos;}

   if (src.substr(0,7) == "SAMPLE ") {
     // Skip SAMPLE ll-Ssss
     pos = SkipOneField(src, pos);
     pos = SkipOneField(src, pos);
   } else if (src.substr(0,5) == "SAMP ") {
     // Skip SAMP ll-Ssss /tld2.tld/
     pos = SkipOneField(src, pos);
     pos = SkipOneField(src, pos);
     pos = SkipOneField(src, pos);
   } else if (src.substr(0,5) == "Samp ") {
     // Skip Samp ll-Ssss /tld2.tld/
     pos = SkipOneField(src, pos);
     pos = SkipOneField(src, pos);
     pos = SkipOneField(src, pos);
   }
   return pos;
 }

 // Avoid zdiv
 inline double Divisor(double x) {
    return (x > 0.0 ? x : 1.0);
 }

 void Flush(Language cur_lang, ULScript ulscript,
            double total_score_cur_lang,
            double total_bytes_cur_lang, double total_bad_bytes_cur_lang) {
   if (cur_lang == UNKNOWN_LANGUAGE) {return;}

   bytes[cur_lang][LScript4(ulscript)] += total_bytes_cur_lang;
   scores[cur_lang][LScript4(ulscript)] += total_score_cur_lang;

   double score = total_score_cur_lang * 1024.0 / Divisor(total_bytes_cur_lang);
   double percent_bad = 100.0 * total_bad_bytes_cur_lang /
      Divisor(total_bytes_cur_lang + total_bad_bytes_cur_lang);
   fprintf(stdout, "%s-%s    %7.0f %6.1f, %2.0f%% bad SUMMARY\n\n",
           LanguageCode(cur_lang),
           ULScriptCode(ulscript),
           total_bytes_cur_lang,
           score,
           percent_bad);
 }

 int BytesPer1KB(int i, int j) {
    int bytes_per_1kb = ((scores[i][j] * 1024.0) / Divisor(bytes[i][j])) + 0.5;
    return bytes_per_1kb;
 }

 int main(int argc, char *argv[]) {
   Language cur_lang = UNKNOWN_LANGUAGE;
   ULScript cur_ulscript = ULScript_Common;
   double total_score_cur_lang = 0.0;
   double total_bytes_cur_lang = 0.0;
   double total_bad_bytes_cur_lang = 0.0;
   memset(bytes, 0, sizeof(bytes));
   memset(scores, 0, sizeof(bytes));

   char buffer[kMaxBuffer];
   int buffer_length;
   const char* filename = NULL;
   FILE* infile = stdin;
   for (int i = 1; i < argc; ++i) {
      if (argv[i][0] != '-') {
         filename = argv[i];
      }
   }

   if (filename != NULL) {
      infile = fopen(filename, "r");
      if (infile == NULL) {
         fprintf(stderr, "%s did not open\n", filename);
         return 0;
      }
   }

   while (ReadLine(infile, buffer, kMaxBuffer)) {
     if (IsComment(buffer)) {continue;}

     buffer_length = strlen(buffer);
     int bytes;
     double score_per_1kb;
     Language toplang;
     Language target_lang;
     ULScript target_ulscript;

     string src(buffer, buffer_length);
     string tld("");
     int pos = GetTextBeginPos(src);
     GetLangScript(src, UNKNOWN_LANGUAGE, ULScript_Common,
                    &target_lang, &target_ulscript, &tld);
     if ((cur_lang != target_lang) || (cur_ulscript != target_ulscript)) {
       Flush(cur_lang, cur_ulscript, total_score_cur_lang,
             total_bytes_cur_lang, total_bad_bytes_cur_lang);
       cur_lang = target_lang;
       cur_ulscript = target_ulscript;
       total_score_cur_lang = 0.0;
       total_bytes_cur_lang = 0.0;
       total_bad_bytes_cur_lang = 0.0;
     }

     toplang = ScoreOneLine(&src[pos], src.size() - pos, &bytes, &score_per_1kb);

     fprintf(stdout, "%s%c %d %4.1f %s\n",
             LanguageCode(toplang),
             (toplang == target_lang) ? ' ' : '*',
             bytes, score_per_1kb, buffer);
     // Only count when detected lang matches the claimed target lang
     if (toplang == target_lang) {
       total_bytes_cur_lang += bytes;
       total_score_cur_lang += (score_per_1kb * bytes) / 1024.0;
     } else {
       total_bad_bytes_cur_lang += bytes;
     }
   }
   Flush(cur_lang, cur_ulscript, total_score_cur_lang,
         total_bytes_cur_lang, total_bad_bytes_cur_lang);

   for (int i = 0; i < NUM_LANGUAGES; ++i) {
      Language ilang = static_cast<Language>(i);
      fprintf(stdout, "  {%4d, %4d, %4d, %4d},  // %d %s %s\n",
              BytesPer1KB(i, 0), BytesPer1KB(i, 1),
              BytesPer1KB(i, 2), BytesPer1KB(i, 3),
              i, LanguageName(ilang), LanguageCode(ilang));
   }

   if (infile != stdin) {
      fclose(infile);
   }
 }
	// Copyright 2013 Google Inc. All Rights Reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	// Little program to read lines of sample text, calculate score per 1024 bytes
	// per language-script4 combination
	// Possible input file /export/hda3/cld/pre2010/b0_samp_prune_20100722.utf8

	#include <stdio.h>
	#include <string.h>
	#include <string>

	#include "compact_lang_det_impl.h"
	#include "lang_script.h"

	using namespace std;
	using namespace CLD2;

	double bytes[NUM_LANGUAGES][4];
	double scores[NUM_LANGUAGES][4];


	// Return score per 1024 bytes for top language
	Language ScoreOneLine(const char* buffer, int buffer_length,
	int* bytes, double* score_per_1kb) {
	bool is_plain_text = true;
	const CLDHints* cld_hints = NULL;
	bool allow_extended_lang = true;
	int flags = 0;
	Language plus_one = UNKNOWN_LANGUAGE;
	Language language3[3];
	int percent3[3];
	double normalized_score3[3];
	ResultChunkVector* resultchunkvector = NULL;
	int text_bytes;
	bool is_reliable;
	Language summary_lang;

	summary_lang = DetectLanguageSummaryV2(
	buffer,
	buffer_length,
	is_plain_text,
	cld_hints,
	allow_extended_lang,
	flags,
	plus_one,
	language3,
	percent3,
	normalized_score3,
	resultchunkvector,
	&text_bytes,
	&is_reliable);
	*bytes = text_bytes;
	*score_per_1kb = normalized_score3[0];
	return language3[0];
	}

	#define LF 0x0a
	#define CR 0x0d
	const int kMaxBuffer = 5 * 1024;

	bool ReadLine(FILE* infile, char* buffer, size_t maxlen) {
	char* p = fgets(buffer, maxlen, infile);
	if (p == NULL) {
	return false;
	}
	int len = strlen(buffer);

	// trim CR LF
	if (buffer[len-1] == LF) {buffer[--len] = '\0';}
	if (buffer[len-1] == CR) {buffer[--len] = '\0';}
	return true;
	}

	bool IsComment(const char* buffer) {
	int len = strlen(buffer);
	if (len == 0) {return true;}
	if (buffer[0] == '#') {return true;}
	if (buffer[0] == ' ') {return true;} // Any leading space is comment
	return false;
	}

	// Skips over xxxxx_ where _ is one or more spaces/tabs
	// Returns string::npos if no more fields
	int SkipOneField(const string& src, int pos) {
	if (pos == string::npos) {return pos;}

	int lpos = pos;
	lpos = src.find_first_of(" \t", lpos);
	if (lpos == string::npos) {return lpos;}
	lpos = src.find_first_not_of(" \t", lpos);
	if (lpos == string::npos) {return lpos;}
	return lpos;
	}

	// Return language and script from parsed line or defaults
	void GetLangScript(const string& src,
	Language default_lang, ULScript default_lscript,
	Language* target_lang, ULScript* target_lscript,
	string* tld) {
	*target_lang = default_lang;
	*target_lscript = default_lscript;
	*tld = "";
	int pos = 0;
	int pos2 = 0;
	if (src.substr(0,7) == "SAMPLE ") {
	// SAMPLE ll-Ssss
	pos = SkipOneField(src, pos);
	} else if (src.substr(0,5) == "SAMP ") {
	// SAMP ll-Ssss /tld2.tld/
	pos = SkipOneField(src, pos);
	pos2 = SkipOneField(src, pos);
	} else if (src.substr(0,5) == "Samp ") {
	// Samp ll-Ssss /tld2.tld/
	pos = SkipOneField(src, pos);
	pos2 = SkipOneField(src, pos);
	}
	if (pos == 0) {return;}
	if (pos == string::npos) {return;}

	// Pos is at the first letter of language-script combination
	int end = src.find_first_of(" \t", pos); // find end of lang-script
	if (end == string::npos) {return;}
	*target_lang = GetLanguageFromName(src.substr(pos, end - pos).c_str());
	*target_lscript = GetULScriptFromName(src.substr(pos, end - pos).c_str());

	// Pos2 is 0 or at the first letter of the tld string
	if (pos2 == 0) {return;}
	if (pos2 == string::npos) {return;}
	end = src.find_first_of(" \t", pos2);
	if (end == string::npos) {return;}
	*tld = src.substr(pos2, end - pos2);
	}

	// Return position of start of text
	int GetTextBeginPos(const string& src) {
	int pos = 0;
	if (src.size() < 8) {return pos;}

	if (src.substr(0,7) == "SAMPLE ") {
	// Skip SAMPLE ll-Ssss
	pos = SkipOneField(src, pos);
	pos = SkipOneField(src, pos);
	} else if (src.substr(0,5) == "SAMP ") {
	// Skip SAMP ll-Ssss /tld2.tld/
	pos = SkipOneField(src, pos);
	pos = SkipOneField(src, pos);
	pos = SkipOneField(src, pos);
	} else if (src.substr(0,5) == "Samp ") {
	// Skip Samp ll-Ssss /tld2.tld/
	pos = SkipOneField(src, pos);
	pos = SkipOneField(src, pos);
	pos = SkipOneField(src, pos);
	}
	return pos;
	}

	// Avoid zdiv
	inline double Divisor(double x) {
	return (x > 0.0 ? x : 1.0);
	}

	void Flush(Language cur_lang, ULScript ulscript,
	double total_score_cur_lang,
	double total_bytes_cur_lang, double total_bad_bytes_cur_lang) {
	if (cur_lang == UNKNOWN_LANGUAGE) {return;}

	bytes[cur_lang][LScript4(ulscript)] += total_bytes_cur_lang;
	scores[cur_lang][LScript4(ulscript)] += total_score_cur_lang;

	double score = total_score_cur_lang * 1024.0 / Divisor(total_bytes_cur_lang);
	double percent_bad = 100.0 * total_bad_bytes_cur_lang /
	Divisor(total_bytes_cur_lang + total_bad_bytes_cur_lang);
	fprintf(stdout, "%s-%s %7.0f %6.1f, %2.0f%% bad SUMMARY\n\n",
	LanguageCode(cur_lang),
	ULScriptCode(ulscript),
	total_bytes_cur_lang,
	score,
	percent_bad);
	}

	int BytesPer1KB(int i, int j) {
	int bytes_per_1kb = ((scores[i][j] * 1024.0) / Divisor(bytes[i][j])) + 0.5;
	return bytes_per_1kb;
	}

	int main(int argc, char *argv[]) {
	Language cur_lang = UNKNOWN_LANGUAGE;
	ULScript cur_ulscript = ULScript_Common;
	double total_score_cur_lang = 0.0;
	double total_bytes_cur_lang = 0.0;
	double total_bad_bytes_cur_lang = 0.0;
	memset(bytes, 0, sizeof(bytes));
	memset(scores, 0, sizeof(bytes));

	char buffer[kMaxBuffer];
	int buffer_length;
	const char* filename = NULL;
	FILE* infile = stdin;
	for (int i = 1; i < argc; ++i) {
	if (argv[i][0] != '-') {
	filename = argv[i];
	}
	}

	if (filename != NULL) {
	infile = fopen(filename, "r");
	if (infile == NULL) {
	fprintf(stderr, "%s did not open\n", filename);
	return 0;
	}
	}

	while (ReadLine(infile, buffer, kMaxBuffer)) {
	if (IsComment(buffer)) {continue;}

	buffer_length = strlen(buffer);
	int bytes;
	double score_per_1kb;
	Language toplang;
	Language target_lang;
	ULScript target_ulscript;

	string src(buffer, buffer_length);
	string tld("");
	int pos = GetTextBeginPos(src);
	GetLangScript(src, UNKNOWN_LANGUAGE, ULScript_Common,
	&target_lang, &target_ulscript, &tld);
	if ((cur_lang != target_lang) \|\| (cur_ulscript != target_ulscript)) {
	Flush(cur_lang, cur_ulscript, total_score_cur_lang,
	total_bytes_cur_lang, total_bad_bytes_cur_lang);
	cur_lang = target_lang;
	cur_ulscript = target_ulscript;
	total_score_cur_lang = 0.0;
	total_bytes_cur_lang = 0.0;
	total_bad_bytes_cur_lang = 0.0;
	}

	toplang = ScoreOneLine(&src[pos], src.size() - pos, &bytes, &score_per_1kb);

	fprintf(stdout, "%s%c %d %4.1f %s\n",
	LanguageCode(toplang),
	(toplang == target_lang) ? ' ' : '*',
	bytes, score_per_1kb, buffer);
	// Only count when detected lang matches the claimed target lang
	if (toplang == target_lang) {
	total_bytes_cur_lang += bytes;
	total_score_cur_lang += (score_per_1kb * bytes) / 1024.0;
	} else {
	total_bad_bytes_cur_lang += bytes;
	}
	}
	Flush(cur_lang, cur_ulscript, total_score_cur_lang,
	total_bytes_cur_lang, total_bad_bytes_cur_lang);

	for (int i = 0; i < NUM_LANGUAGES; ++i) {
	Language ilang = static_cast<Language>(i);
	fprintf(stdout, " {%4d, %4d, %4d, %4d}, // %d %s %s\n",
	BytesPer1KB(i, 0), BytesPer1KB(i, 1),
	BytesPer1KB(i, 2), BytesPer1KB(i, 3),
	i, LanguageName(ilang), LanguageCode(ilang));
	}

	if (infile != stdin) {
	fclose(infile);
	}
	}