Java中N-gram词组频率统计与最常见短语提取教程

// 示例输入数据结构
ArrayList<ArrayList<String>> sentence = new ArrayList<>();
// 模拟一些预处理后的句子
ArrayList<String> s1 = new ArrayList<>(Arrays.asList("this", "is", "a", "sample", "sentence"));
ArrayList<String> s2 = new ArrayList<>(Arrays.asList("another", "sample", "text", "for", "analysis"));
ArrayList<String> s3 = new ArrayList<>(Arrays.asList("this", "is", "another", "example"));
sentence.add(s1);
sentence.add(s2);
sentence.add(s3);

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

public class NGramAnalyzer {

    /**
     * 统计所有Bigram的频率。
     * @param sentences 预处理后的句子列表，每个句子是一个词语列表。
     * @return 包含所有Bigram及其频率的HashMap。
     */
    public static HashMap<String, Integer> countBigrams(ArrayList<ArrayList<String>> sentences) {
        HashMap<String, Integer> nGramMap = new HashMap<>();

        // 遍历每个句子
        for (ArrayList<String> words : sentences) {
            // 遍历句子中的词语，生成连续的词对 (Bigrams)
            // 注意循环条件：i < words.size() - 1，确保 i+1 不越界
            for (int i = 0; i < words.size() - 1; i++) {
                // 组合当前词和下一个词形成Bigram
                String nGram = words.get(i) + " " + words.get(i + 1);

                // 更新Bigram的频率
                nGramMap.put(nGram, nGramMap.getOrDefault(nGram, 0) + 1);
            }
        }
        return nGramMap;
    }

    // ... 后续代码用于查找最常见的N-gram
}

// 承接上文的 NGramAnalyzer 类

public class NGramAnalyzer {

    // ... countBigrams 方法如上所示 ...

    /**
     * 从Bigram频率映射表中找出出现频率最高的Bigram。
     * @param nGramMap 包含Bigram及其频率的HashMap。
     * @return 出现频率最高的Bigram字符串。如果映射表为空，返回空字符串。
     */
    public static String findMostCommonNGram(HashMap<String, Integer> nGramMap) {
        String mostCommonNGram = "";
        int maxCount = 0;

        if (nGramMap.isEmpty()) {
            return mostCommonNGram; // 处理空映射表的情况
        }

        // 遍历HashMap的entrySet，查找最大计数
        for (Map.Entry<String, Integer> entry : nGramMap.entrySet()) {
            String nGram = entry.getKey();
            int count = entry.getValue();

            if (count > maxCount) {
                maxCount = count;
                mostCommonNGram = nGram;
            }
        }
        return mostCommonNGram;
    }

    public static void main(String[] args) {
        // 示例输入数据
        ArrayList<ArrayList<String>> sentences = new ArrayList<>();
        sentences.add(new ArrayList<>(Arrays.asList("this", "is", "a", "sample", "sentence")));
        sentences.add(new ArrayList<>(Arrays.asList("another", "sample", "text", "for", "analysis")));
        sentences.add(new ArrayList<>(Arrays.asList("this", "is", "another", "example")));
        sentences.add(new ArrayList<>(Arrays.asList("sample", "sentence", "is", "important"))); // 添加更多数据以增加重复

        // 1. 统计所有Bigram的频率
        HashMap<String, Integer> bigramFrequencies = countBigrams(sentences);
        System.out.println("所有Bigram及其频率: " + bigramFrequencies);

        // 2. 查找最常见的Bigram
        String mostCommon = findMostCommonNGram(bigramFrequencies);
        System.out.println("最常见的Bigram是: \"" + mostCommon + "\"");
        System.out.println("出现次数: " + bigramFrequencies.getOrDefault(mostCommon, 0));
    }
}

// 泛化到任意N-gram的生成
public static HashMap<String, Integer> countNGrams(ArrayList<ArrayList<String>> sentences, int n) {
    HashMap<String, Integer> nGramMap = new HashMap<>();

    for (ArrayList<String> words : sentences) {
        if (words.size() < n) { // 句子长度不足以构成N-gram
            continue;
        }
        for (int i = 0; i <= words.size() - n; i++) {
            StringBuilder nGramBuilder = new StringBuilder();
            for (int j = 0; j < n; j++) {
                nGramBuilder.append(words.get(i + j));
                if (j < n - 1) {
                    nGramBuilder.append(" "); // 用空格分隔词语
                }
            }
            String nGram = nGramBuilder.toString();
            nGramMap.put(nGram, nGramMap.getOrDefault(nGram, 0) + 1);
        }
    }
    return nGramMap;
}