1/*
2 * Copyright (C) 2007 Google Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package com.android.voicedialer;
18
19import android.content.Intent;
20import android.util.Log;
21import java.io.File;
22import java.io.FileFilter;
23import java.util.Arrays;
24import java.util.HashSet;
25import java.util.Set;
26import java.util.Vector;
27
28/**
29 * This class represents a person who may be called via the VoiceDialer app.
30 * The person has a name and a list of phones (home, mobile, work).
31 */
32public class VoiceDialerTester {
33    private static final String TAG = "VoiceDialerTester";
34
35    private final WavFile[] mWavFiles;
36    private final File[] mWavDirs;
37
38    // these indicate the current test
39    private int mWavFile = -1; // -1 so it will step to 0 on first iteration
40
41    private static class WavFile {
42        final public File mFile;
43        public int mRank;
44        public int mTotal;
45        public String mMessage;
46
47        public WavFile(File file) {
48            mFile = file;
49        }
50    }
51
52    /**
53     * Sweep directory of directories, listing all WAV files.
54     */
55    public VoiceDialerTester(File dir) {
56        if (false) {
57            Log.d(TAG, "VoiceDialerTester " + dir);
58        }
59
60        // keep a list of directories visited
61        Vector<File> wavDirs = new Vector<File>();
62        wavDirs.add(dir);
63
64        // scan the directory tree
65        Vector<File> wavFiles = new Vector<File>();
66        for (int i = 0; i < wavDirs.size(); i++) {
67            File d = wavDirs.get(i);
68            for (File f : d.listFiles()) {
69                if (f.isFile() && f.getName().endsWith(".wav")) {
70                    wavFiles.add(f);
71                }
72                else if (f.isDirectory()) {
73                    wavDirs.add(f);
74                }
75            }
76        }
77
78        // produce a sorted list of WavFiles
79        File[] fa = wavFiles.toArray(new File[wavFiles.size()]);
80        Arrays.sort(fa);
81        mWavFiles = new WavFile[fa.length];
82        for (int i = 0; i < mWavFiles.length; i++) {
83            mWavFiles[i] = new WavFile(fa[i]);
84        }
85
86        // produce a sorted list of directories
87        mWavDirs = wavDirs.toArray(new File[wavDirs.size()]);
88        Arrays.sort(mWavDirs);
89    }
90
91    public File getWavFile() {
92        return mWavFiles[mWavFile].mFile;
93    }
94
95    /**
96     * Called by VoiceDialerActivity when a recognizer error occurs.
97     */
98    public void onRecognitionError(String msg) {
99        WavFile wf = mWavFiles[mWavFile];
100        wf.mRank = -1;
101        wf.mTotal = -1;
102        wf.mMessage = msg;
103    }
104
105    /**
106     * Called by VoiceDialerActivity when a recognizer failure occurs.
107     * @param msg Message to display.
108     */
109    public void onRecognitionFailure(String msg) {
110        WavFile wf = mWavFiles[mWavFile];
111        wf.mRank = 0;
112        wf.mTotal = 0;
113        wf.mMessage = msg;
114    }
115
116    /**
117     * Called by VoiceDialerActivity when the recognizer succeeds.
118     * @param intents Array of Intents corresponding to recognized sentences.
119     */
120    public void onRecognitionSuccess(Intent[] intents) {
121        WavFile wf = mWavFiles[mWavFile];
122        wf.mTotal = intents.length;
123        String utter = wf.mFile.getName().toLowerCase().replace('_', ' ');
124        utter = utter.substring(0, utter.indexOf('.')).trim();
125        for (int i = 0; i < intents.length; i++) {
126            String sentence =
127                    intents[i].getStringExtra(RecognizerEngine.SENTENCE_EXTRA).
128                    toLowerCase().trim();
129            // note the first in case there are no matches
130            if (i == 0) {
131                wf.mMessage = sentence;
132                if (intents.length > 1) wf.mMessage += ", etc";
133            }
134            // is this a match
135            if (utter.equals(sentence)) {
136                wf.mRank = i + 1;
137                wf.mMessage = null;
138                break;
139            }
140        }
141    }
142
143    /**
144     * Called to step to the next WAV file in the test set.
145     * @return true if successful, false if no more test files.
146     */
147    public boolean stepToNextTest() {
148        mWavFile++;
149        return mWavFile < mWavFiles.length;
150    }
151
152    private static final String REPORT_FMT = "%6s %6s %6s %6s %6s %6s %6s %s";
153    private static final String REPORT_HDR = String.format(REPORT_FMT,
154            "1/1", "1/N", "M/N", "0/N", "Fail", "Error", "Total", "");
155
156    /**
157     * Called when the test is complete to dump a summary.
158     */
159    public void report() {
160        // report for each file
161        Log.d(TAG, "List of all utterances tested");
162        for (WavFile wf : mWavFiles) {
163            Log.d(TAG, wf.mRank + "/" + wf.mTotal + "  " + wf.mFile +
164                    (wf.mMessage != null ? "  " + wf.mMessage : ""));
165        }
166
167        // summary reports by file name
168        reportSummaryForEachFileName();
169
170        // summary reports by directory name
171        reportSummaryForEachDir();
172
173        // summary report for all files
174        Log.d(TAG, "Summary of all utterances");
175        Log.d(TAG, REPORT_HDR);
176        reportSummary("Total", null);
177    }
178
179    private void reportSummaryForEachFileName() {
180        Set<String> set = new HashSet<String>();
181        for (WavFile wf : mWavFiles) {
182            set.add(wf.mFile.getName());
183        }
184        String[] names = set.toArray(new String[set.size()]);
185        Arrays.sort(names);
186
187        Log.d(TAG, "Summary of utternaces by filename");
188        Log.d(TAG, REPORT_HDR);
189        for (final String fn : names) {
190            reportSummary(fn,
191                    new FileFilter() {
192                        public boolean accept(File file) {
193                            return fn.equals(file.getName());
194                        }
195            });
196        }
197    }
198
199    private void reportSummaryForEachDir() {
200        Set<String> set = new HashSet<String>();
201        for (WavFile wf : mWavFiles) {
202            set.add(wf.mFile.getParent());
203        }
204        String[] names = set.toArray(new String[set.size()]);
205        Arrays.sort(names);
206
207        Log.d(TAG, "Summary of utterances by directory");
208        Log.d(TAG, REPORT_HDR);
209        for (File dir : mWavDirs) {
210            final String dn = dir.getPath();
211            final String dn2 = dn + "/";
212            reportSummary(dn,
213                    new FileFilter() {
214                        public boolean accept(File file) {
215                            return file.getPath().startsWith(dn2);
216                        }
217            });
218        }
219    }
220
221    private void reportSummary(String label, FileFilter filter) {
222        if (!false) return;
223
224        // log cumulative stats
225        int total = 0;
226        int count11 = 0;
227        int count1N = 0;
228        int countMN = 0;
229        int count0N = 0;
230        int countFail = 0;
231        int countErrors = 0;
232
233        for (WavFile wf : mWavFiles) {
234            if (filter == null || filter.accept(wf.mFile)) {
235                total++;
236                if (wf.mRank == 1 && wf.mTotal == 1) count11++;
237                if (wf.mRank == 1 && wf.mTotal >= 1) count1N++;
238                if (wf.mRank >= 1 && wf.mTotal >= 1) countMN++;
239                if (wf.mRank == 0 && wf.mTotal >= 1) count0N++;
240                if (wf.mRank == 0 && wf.mTotal == 0) countFail++;
241                if (wf.mRank == -1 && wf.mTotal == -1) countErrors++;
242            }
243        }
244
245        String line = String.format(REPORT_FMT,
246                countString(count11, total),
247                countString(count1N, total),
248                countString(countMN, total),
249                countString(count0N, total),
250                countString(countFail, total),
251                countString(countErrors, total),
252                "" + total,
253                label);
254        Log.d(TAG, line);
255    }
256
257    private static String countString(int count, int total) {
258        return total > 0 ? "" + (100 * count / total) + "%" : "";
259    }
260
261}
262