@Article{puketza96, author = {N. J. Puketza and K. Zhang and M. Chung and B. Mukherjee and {R. A.} Olsson}, title = {A Methodology for Testing Intrusion Detection Systems}, journal = {IEEE Transactions on Software Engineering}, year = 1996, volume = 22, number = 10 } @InProceedings{lippmann98, author = {{R. P.} Lippmann and { D. J.} Fried and I. Graf and {J. W.} Haines and {K. P.} Kendall and D. McClung and D. Weber and {S. E.} Webster and D. Wyschogrod and { R. K.} Cunningham and {M. A.} Zissman}, title = {Evaluating Intrusion Detection Systems: The 1998 DARPA Off-line Intrusion Detection Evaluation}, booktitle = {Proceedings of the 2000 DARPA Information Survivability Conference and Exposition}, year = 2000, organization = {DARPA} } @TechReport{lippmann99, author = {{R. P.} Lippmann and {J. W.} Haines and {D. J.} Fried and J. Korba and K. Das}, title = {The 1999 DARPA Off-line Intrusion Detection Evaluation}, institution = {MIT Lincoln Lab}, year = 2000 } @Article{mchugh00, author = {J. McHugh}, title = {Testing Intrusion Detection Systems: A Critique of the 1998 and 1999 DARPA Off-line Intrusion Detection System Evaluation as Performed by Lincoln Laboratory}, journal = {ACM Transactions on Information and System Security}, year = 2000, volume = 3, number = 4, month = {November} } @Book{grant-spc-72, author = {{E.L.} Grant and {R.S} Leavenworth}, title = {Statistical Quality Control}, publisher = {McGraw-Hill }, year = 1972 } @Article{puketza97, author = {N. Puketza and M. Chung and { R. A.} Olsson and B. Mukherjee}, title = {A Software Platform for Testing Intrusion Detection Systems}, journal = {IEEE Software}, year = 1997, pages = {43-51}, month = {September/October} } @InProceedings{amoroso-98, author = { E. Amoroso and R. Kwapniewski}, title = {A selection criteria for intrusion detection systems}, booktitle = {Proc. 14th AnnualComputer Security Applications Conference}, pages = {280 - 288}, year = 1998, month = {Dec}, organization = {IEEE} } @Article{lippmann00, author = {R. Lippmann and J. Haines and D. Fried and J. Korba and K. Das}, title = {The 1999 DARPA off-line intrusion detection evaluation}, journal = {Computer Networks}, year = 2000, volume = 34, pages = { 579-595} } @InProceedings{maxion00, author = {{R.A.} Maxion and {K.M.C.} Tan}, title = {Benchmarking anomaly-based detection systems}, booktitle = {IEEE Proc Int. Conf on Dependable Systems and Networks}, pages = {623-630}, year = 2000 } @InProceedings{lariat, author = {{ L.M.} Rossey and { R.K.} Cunningham and { D.J.} Fried and { J.C.} Rabek and { R.P.} Lippmann and { J.W.} Haines and { M.A.} Zissman}, title = {LARIAT: Lincoln adaptable real-time information assurance testbed}, booktitle = {IEEE Proc. Aerospace Conference}, pages = {2671-2682}, year = 2002, volume = 6, month = {March} } @InProceedings{toth-02, author = {T. Toth and C. Kruegel}, title = {Evaluating the impact of automated intrusion response mechanisms}, booktitle = {18th Computer Security Applications Conference}, pages = {301-310}, year = 2002, month = {December}, organization = {IEEE} } @InProceedings{fink-02, author = {{ G.A.} Fink and{ B.L.} Chappell and{ T.G.} Turner and{ K.F.} O'Donoghue }, title = {A metrics-based approach to intrusion detection system evaluation for distributed real-time systems}, booktitle = {Proc. Int. Parallel and Distributed Processing Symposium}, pages = {93-100}, year = 2002, organization = {IEEE} } @InProceedings{athanasiades-03, author = {{ N.} Athanasiades and{ R.} Abler and{ J.} Levine and{ H.} Owen and{ G.} Riley}, title = {Intrusion detection testing and benchmarking methodologies}, booktitle = {First IEEE Int. Workshop on Informatoin Assurance (IWIAS)}, pages = {63-72}, year = 2003, month = {March}, organization = {IEEE} } @InProceedings{allen-03, author = {{ W.H.} Allen and{ G.A.} Marin}, title = {On the self-similarity of synthetic traffic for the evaluation of intrusion detection systems}, booktitle = {Proc. Symp. on Applications and the Internet}, pages = {242-248}, year = 2003, month = {Jan}, organization = {IEEE} } @Article{tan-03, author = {{ K.M.C.} Tan and{ R.A.} Maxion}, title = {Determining the operational limits of an anomaly-based intrusion detector}, journal = {IEEE Journal on Selected Areas in Communications}, year = 2003, volume = 21, number = 1, pages = {96-110}, month = {Jan} } @InProceedings{murray-spencer-94, author = {{ D.W.} Murray and{ D.D.} Spencer}, title = {Statistical process control testing of electronic security equipment}, booktitle = {IEEE Int. Carnahan Conf on Security Technology}, pages = {53-59}, year = 1994, month = {Oct}, organization = {IEEE} } @MastersThesis{korba00, author = {J. Korba}, title = {Windows NT Attacks for the Evaluation of Intrusion Detection Systems}, school = {MIT EECS}, year = 2000, month = {June} } @MastersThesis{das00, author = {K. Das}, title = {The Development of Stealthy Attacks to Evaluate Intrusion Detection Systems}, school = {MIT EECS}, year = 2000, month = {June} } @MastersThesis{kendall99, author = {K. Kendall}, title = {A Database of Computer Attacks for the Evaluation of Intrusion Detection Systems}, school = {MIT EECS}, year = 1999 } @InProceedings{alessandri00, author = {D. Alessandri}, title = {Using Rule-Based Activity Descriptions to Evaluate Intrusion Detection Systems}, booktitle = {Int. Workshop on Recent Advances in Intrusion Detection}, pages = {183-196}, year = 2000, editor = {H. Debar and L. Me and {S. F.} Wu}, volume = 1907, series = {Lectures in CS}, publisher = {Springer Verlag} } @TechReport{aguirre97, author = {{S.J.} Aguirre and {W.H.}Hill }, title = {Intrusion Detection Fly-Off: Implications for the United States Navy}, institution = {MITRE }, year = {1997}, OPTkey = {}, OPTtype = {}, OPTnumber = {Technical Report MTR 97W096}, OPTaddress = {McLean, Virginia. }, OPTmonth = {September}, OPTnote = {}, OPTannote = {} } @Article{neophpsis-99a, author = {G. Shipley}, title = {ISS RealSecure Pushes Past Newer IDS Players}, journal = {Network Computing}, year = 1999, month = {May}, note = {http://www.networkcomputing.com/1010/1010r1.html} } @Article{neophpsis-99b, author = {G. Shipley}, title = {Intrusion Detection, Take Two}, journal = {Network Computing}, year = 1999, month = {November}, note = {http://www.networkcomputing.com/1023/1023f1.html} } @Article{mueller01, author = {P. Mueller and G. Shipley}, title = {Dragon claws its way to the top}, journal = {Network Computing}, year = 2001, month = {August}, note = {http://www.networkcomputing.com/1217/1217f2.html} } @InProceedings{snort, author = {M. Roesch }, title = {Snort - Lightweight Intrusion Detection for Networks}, booktitle = {USENIX 13th Systems Administration Conference - LISA '99}, year = 1999, address = {Seattle, Washington}, organization = {usenix}, note = {see also www.snort.org} } @misc{Nlar, author = {National Laboratory for Applied Network Research}, title = {NLAR network traffic packet header traces}, year = 2002, note = {http://pma.nlanr.net/Traces/} } @InProceedings{mahoney03, author = {{M.V.} Mahoney and {P.K.} Chan }, title = {An Analysis of the 1999 DARPA/Lincond Laboratory Evaluation Data for Network Anomaly Detection}, booktitle = {Proc. Recent Advances in Intrusion Detection}, pages = {220-237}, series = {Lectures in CS}, publisher = {Springer Verlag}, year = {2003}, volume = {2820}, month = {November} } @InProceedings{anzen, author = {D. Song and G. Shaffer and M. Undy}, title = {Nidsbench - A network intrusion detection test suite}, booktitle = {Recent Advances in Intrusion Detection, Second International Workshop}, year = 1999, address = {West Lafayette}, note = {http://www.raid-symposium.org/raid99/PAPERS/Song.pdf} } @InProceedings{beverge-cofactors, author = {G. Givens and {J.R.} Beveridge and {B.A.} Draper and D. Bolme }, title = {A Statistical Assessment of Subject Factors in the PCA Recognition of Human Faces}, booktitle = {"IEEE Workshop on Statistical Analysis in Computer Vision}, year = 2003, month = {June}, organization = {IEEE} } @InProceedings{champion01, author = {T. Champion and M. Denz}, title = {A Benchmark Evaluation of Network Intrusion Detection Systems}, booktitle = {Proc. of IEEE Conf. on Aerospace Systems}, year = 2001 } @InProceedings{stolfo-00, author = {S. Stolfo and W. Fan and W. Lee and A. Prodromidis and P. Chan}, title = {Cost-based Modeling for Fraud and Intrusion Detection: Results from the JAM Project}, booktitle = {n Proceedings of the 2000 DARPA Information Survivability Conference and Exposition (DISCEX '00)}, year = 2000 } @InProceedings{gaffney-01, author = {{J.E.} Gaffney and {J.W. } Ulvila}, title = {Evaluation of Intrusion Detectors: A decision Theory Approach}, booktitle = {IEEE Symp. on Security and Privacy}, year = 2001, address = {Oakland, CA}, month = {May}, organization = {IEEE} } @InProceedings{carver01, author = {{C. A.} Carver and {J. M. D.} Hill and {U. W.} Pooch}, title = {Limiting Uncertainty in Intrusion Response}, booktitle = {Proc. of IEEE Workshop on Information Assurance and Security}, year = 2001, address = {US Military Academy, West Point}, month = {June} } @InProceedings{porras97:_emeral, author = {{P. A.} Porras and {P. G.} Neumann}, title = {Emerald: Event monitoring enabling responses to anomalous live disturbances}, booktitle = {Proceedings of the 20th NIS Security Conference}, year = 1997, month = {October} } @InProceedings{sterne01, author = {D. Sterne and K. Djahandari and B. Wilson and B. Babson and D. Schnackenberg and H. Holliday and T. Reid}, title = {Autonomic Response to Distributed Denial of Service Attacks}, booktitle = {Proc. International Symp on Rapid Advances in Intrusion Detection (RAID)}, year = 2001, address = {Davis, CA, USA}, month = {October} } @InProceedings{balepin03, author = {I. Balepin and S. Maltsev and J. Rowe and K Levitt}, title = {Using Specification-Based Intrusion Detection for Automated Responce}, booktitle = {Proc. International Symp on Rapid Advances in Intrusion Detection (RAID)}, year = 2003, address = {Pittsburg, PA, USA}, month = {September} } @InProceedings{micheals-boult-cvpr, author = {R. J. Micheals and T. E. Boult}, title = {Efficient Evaluation of Classification and Recognition Systems }, booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR 2001)}, address = {Hawaii}, month = {December 11--13}, year = {2001} } @Article{feret, author = {P. J. Phillips and H. Moon and S. A. Rizvi and P. J. Rauss}, title = {The \hbox{FERET} Evaluation Methodology for Face-Recognition Algorithms}, journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence}, year = {2000}, volume = {22}, number = {10}, pages = {1090--1104}, month = {October}, } @Article{sitter, author = {R. R. Sitter}, title = {Balanced Repeated Replications Based on Orthogonal Multi-Arrays}, journal = {Biometrika}, year = {1993}, volume = {80}, number = {1}, pages = {211-221}, month = {March}, } @Article{shao_wu, author = {J. Shao and C. F. J. Wu}, title = {Asymptotic Properties of the Balanced Repeated Replication Method for Sample Quantiles}, journal = {Annals of Statistics}, year = {1992}, volume = {20}, number = {3}, pages = {1571--1593}, month = {September}, } @Article{krewski_rao, author = {D. Krewski and J. N. K. Rao}, title = {Inference From Statified Samples: Properties of the Linearization, Jackknife and Balanced Repeated Replication Methods}, journal = {The Annals of Statistics}, year = {1981}, volume = {9}, number = {5}, pages = {1010--1019}, } @InProceedings{micheals03, author = {{R. J.} Micheals and P. Grother and {P.J.} Phillips}, title = {The NIST Human ID Evaluation Framework}, booktitle = {Proc. of the 4th Int. Conference on Audio and Video-based Biometric Person Authentication}, year = 2003, month = {June}, note = {Availible from http://www.frvt.org/DLs/AVBPA-2003.pdf} } @TechReport{frvt2002, author = {{ P.J.} Phillips and P. Grother and {R. J.} Micheals and {D. M.} Blackburn and E. Tabassi and M. Bone}, title = {Face Recognition Vendor Test 2002. Evaluation Report}, institution = {National Institute of Standards and Technology}, year = 2003, number = {IR 6965}, month = {March}, note = {www.itl.nist.gov/iad/894.03/face/face.html} } @InProceedings{grother03, author = {{P.J.} Grother and {R.J.} Micheals and {P. J.} Phillips}, title = {Face Recognition Vendor Test 2002 Performance Metrics}, booktitle = {Proceedings 4th International Conference on Audio Visual Based Person Authentication}, year = 2003, month = {June} } @PhdThesis{micheals-thesis, author = {{R.J.} Micheals}, title = {Biometric systems evaluation}, school = {Lehigh University}, year = 2003 } @TechReport{micheals-urn-tr, author = {{R.J.} Micheals and {T.E.} Boult}, title = {Is the Urn Well-Mixed?}, institution = {National Institute of Standards and Technology}, year = 2004, month = {Febuary}, } @InProceedings{micheals-strat, author = {{R. J.} Micheals and {T. E.} Boult}, title = {A Stratified Methodology for Classifier and Recognizer Evaluation}, booktitle = {IEEE Workshop on Emperical Evaluation Methods in Computer Vision}, year = 2001, address = {Kauai, Hawaii}, month = {Dec}, organization = {IEEE} }