[{"language":[{"iso":"eng"}],"oa":1,"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","citation":{"ama":"Kueffner K, Lukina A, Schilling C, Henzinger TA. Into the unknown: Active monitoring of neural networks (extended version). <i>International Journal on Software Tools for Technology Transfer</i>. 2023;25:575-592. doi:<a href=\"https://doi.org/10.1007/s10009-023-00711-4\">10.1007/s10009-023-00711-4</a>","ieee":"K. Kueffner, A. Lukina, C. Schilling, and T. A. Henzinger, “Into the unknown: Active monitoring of neural networks (extended version),” <i>International Journal on Software Tools for Technology Transfer</i>, vol. 25. Springer Nature, pp. 575–592, 2023.","short":"K. Kueffner, A. Lukina, C. Schilling, T.A. Henzinger, International Journal on Software Tools for Technology Transfer 25 (2023) 575–592.","ista":"Kueffner K, Lukina A, Schilling C, Henzinger TA. 2023. Into the unknown: Active monitoring of neural networks (extended version). International Journal on Software Tools for Technology Transfer. 25, 575–592.","chicago":"Kueffner, Konstantin, Anna Lukina, Christian Schilling, and Thomas A Henzinger. “Into the Unknown: Active Monitoring of Neural Networks (Extended Version).” <i>International Journal on Software Tools for Technology Transfer</i>. Springer Nature, 2023. <a href=\"https://doi.org/10.1007/s10009-023-00711-4\">https://doi.org/10.1007/s10009-023-00711-4</a>.","apa":"Kueffner, K., Lukina, A., Schilling, C., &#38; Henzinger, T. A. (2023). Into the unknown: Active monitoring of neural networks (extended version). <i>International Journal on Software Tools for Technology Transfer</i>. Springer Nature. <a href=\"https://doi.org/10.1007/s10009-023-00711-4\">https://doi.org/10.1007/s10009-023-00711-4</a>","mla":"Kueffner, Konstantin, et al. “Into the Unknown: Active Monitoring of Neural Networks (Extended Version).” <i>International Journal on Software Tools for Technology Transfer</i>, vol. 25, Springer Nature, 2023, pp. 575–92, doi:<a href=\"https://doi.org/10.1007/s10009-023-00711-4\">10.1007/s10009-023-00711-4</a>."},"month":"08","arxiv":1,"file":[{"date_created":"2024-01-30T12:06:07Z","file_size":13387667,"date_updated":"2024-01-30T12:06:07Z","creator":"dernst","file_id":"14903","file_name":"2023_JourSoftwareTools_Kueffner.pdf","success":1,"content_type":"application/pdf","access_level":"open_access","relation":"main_file","checksum":"3c4b347f39412a76872f9a6f30101f94"}],"department":[{"_id":"ToHe"}],"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","image":"/images/cc_by.png","short":"CC BY (4.0)"},"abstract":[{"lang":"eng","text":"Neural-network classifiers achieve high accuracy when predicting the class of an input that they were trained to identify. Maintaining this accuracy in dynamic environments, where inputs frequently fall outside the fixed set of initially known classes, remains a challenge. We consider the problem of monitoring the classification decisions of neural networks in the presence of novel classes. For this purpose, we generalize our recently proposed abstraction-based monitor from binary output to real-valued quantitative output. This quantitative output enables new applications, two of which we investigate in the paper. As our first application, we introduce an algorithmic framework for active monitoring of a neural network, which allows us to learn new classes dynamically and yet maintain high monitoring performance. As our second application, we present an offline procedure to retrain the neural network to improve the monitor’s detection performance without deteriorating the network’s classification accuracy. Our experimental evaluation demonstrates both the benefits of our active monitoring framework in dynamic scenarios and the effectiveness of the retraining procedure."}],"intvolume":"        25","has_accepted_license":"1","file_date_updated":"2024-01-30T12:06:07Z","publication_identifier":{"eissn":["1433-2787"],"issn":["1433-2779"]},"publication_status":"published","title":"Into the unknown: Active monitoring of neural networks (extended version)","oa_version":"Published Version","scopus_import":"1","day":"01","author":[{"first_name":"Konstantin","orcid":"0000-0001-8974-2542","full_name":"Kueffner, Konstantin","id":"8121a2d0-dc85-11ea-9058-af578f3b4515","last_name":"Kueffner"},{"last_name":"Lukina","id":"CBA4D1A8-0FE8-11E9-BDE6-07BFE5697425","full_name":"Lukina, Anna","first_name":"Anna"},{"full_name":"Schilling, Christian","id":"3A2F4DCE-F248-11E8-B48F-1D18A9856A87","last_name":"Schilling","first_name":"Christian","orcid":"0000-0003-3658-1065"},{"orcid":"0000-0002-2985-7724","first_name":"Thomas A","id":"40876CD8-F248-11E8-B48F-1D18A9856A87","full_name":"Henzinger, Thomas A","last_name":"Henzinger"}],"date_created":"2023-07-16T22:01:11Z","article_type":"original","volume":25,"status":"public","publication":"International Journal on Software Tools for Technology Transfer","project":[{"_id":"62781420-2b32-11ec-9570-8d9b63373d4d","grant_number":"101020093","name":"Vigilant Algorithmic Monitoring of Software","call_identifier":"H2020"}],"acknowledgement":"This work was supported in part by the ERC-2020-AdG 101020093, by DIREC - Digital Research Centre Denmark, and by the Villum Investigator Grant S4OS.","date_published":"2023-08-01T00:00:00Z","ec_funded":1,"related_material":{"record":[{"id":"10206","relation":"shorter_version","status":"public"}]},"external_id":{"arxiv":["2009.06429"],"isi":["001020160000001"]},"isi":1,"year":"2023","ddc":["000"],"page":"575-592","quality_controlled":"1","publisher":"Springer Nature","article_processing_charge":"Yes (in subscription journal)","doi":"10.1007/s10009-023-00711-4","type":"journal_article","_id":"13234","date_updated":"2024-01-30T12:06:57Z"},{"main_file_link":[{"url":"https://arxiv.org/abs/2009.06429","open_access":"1"}],"quality_controlled":"1","page":"42-61","_id":"10206","date_updated":"2024-01-30T12:06:56Z","type":"conference","article_processing_charge":"No","alternative_title":["LNCS"],"doi":"10.1007/978-3-030-88494-9_3","publisher":"Springer Nature","ec_funded":1,"acknowledgement":"We thank Christoph Lampert and Alex Greengold for fruitful discussions. This research was supported in part by the Simons Institute for the Theory of Computing, the Austrian Science Fund (FWF) under grant Z211-N23 (Wittgenstein Award), and the European Union’s Horizon 2020 research and innovation programme under the Marie Skłodowska-Curie grant agreement No. 754411.","conference":{"location":"Virtual","name":"RV: Runtime Verification","start_date":"2021-10-11","end_date":"2021-10-14"},"date_published":"2021-10-06T00:00:00Z","status":"public","publication":"21st International Conference on Runtime Verification","project":[{"_id":"260C2330-B435-11E9-9278-68D0E5697425","call_identifier":"H2020","grant_number":"754411","name":"ISTplus - Postdoctoral Fellowships"},{"_id":"25F42A32-B435-11E9-9278-68D0E5697425","name":"The Wittgenstein Prize","grant_number":"Z211","call_identifier":"FWF"}],"keyword":["monitoring","neural networks","novelty detection"],"year":"2021","isi":1,"related_material":{"record":[{"relation":"extended_version","status":"public","id":"13234"}]},"external_id":{"isi":["000719383800003"],"arxiv":["2009.06429"]},"publication_status":"published","publication_identifier":{"eissn":["1611-3349"],"eisbn":["978-3-030-88494-9"],"issn":["0302-9743"],"isbn":["9-783-0308-8493-2"]},"abstract":[{"text":"Neural-network classifiers achieve high accuracy when predicting the class of an input that they were trained to identify. Maintaining this accuracy in dynamic environments, where inputs frequently fall outside the fixed set of initially known classes, remains a challenge. The typical approach is to detect inputs from novel classes and retrain the classifier on an augmented dataset. However, not only the classifier but also the detection mechanism needs to adapt in order to distinguish between newly learned and yet unknown input classes. To address this challenge, we introduce an algorithmic framework for active monitoring of a neural network. A monitor wrapped in our framework operates in parallel with the neural network and interacts with a human user via a series of interpretable labeling queries for incremental adaptation. In addition, we propose an adaptive quantitative monitor to improve precision. An experimental evaluation on a diverse set of benchmarks with varying numbers of classes confirms the benefits of our active monitoring framework in dynamic scenarios.","lang":"eng"}],"volume":"12974 ","date_created":"2021-10-31T23:01:31Z","day":"06","scopus_import":"1","author":[{"first_name":"Anna","last_name":"Lukina","full_name":"Lukina, Anna","id":"CBA4D1A8-0FE8-11E9-BDE6-07BFE5697425"},{"first_name":"Christian","orcid":"0000-0003-3658-1065","full_name":"Schilling, Christian","id":"3A2F4DCE-F248-11E8-B48F-1D18A9856A87","last_name":"Schilling"},{"last_name":"Henzinger","id":"40876CD8-F248-11E8-B48F-1D18A9856A87","full_name":"Henzinger, Thomas A","orcid":"0000-0002-2985-7724","first_name":"Thomas A"}],"oa_version":"Preprint","title":"Into the unknown: active monitoring of neural networks","citation":{"ama":"Lukina A, Schilling C, Henzinger TA. Into the unknown: active monitoring of neural networks. In: <i>21st International Conference on Runtime Verification</i>. Vol 12974. Cham: Springer Nature; 2021:42-61. doi:<a href=\"https://doi.org/10.1007/978-3-030-88494-9_3\">10.1007/978-3-030-88494-9_3</a>","ieee":"A. Lukina, C. Schilling, and T. A. Henzinger, “Into the unknown: active monitoring of neural networks,” in <i>21st International Conference on Runtime Verification</i>, Virtual, 2021, vol. 12974, pp. 42–61.","short":"A. Lukina, C. Schilling, T.A. Henzinger, in:, 21st International Conference on Runtime Verification, Springer Nature, Cham, 2021, pp. 42–61.","ista":"Lukina A, Schilling C, Henzinger TA. 2021. Into the unknown: active monitoring of neural networks. 21st International Conference on Runtime Verification. RV: Runtime Verification, LNCS, vol. 12974, 42–61.","chicago":"Lukina, Anna, Christian Schilling, and Thomas A Henzinger. “Into the Unknown: Active Monitoring of Neural Networks.” In <i>21st International Conference on Runtime Verification</i>, 12974:42–61. Cham: Springer Nature, 2021. <a href=\"https://doi.org/10.1007/978-3-030-88494-9_3\">https://doi.org/10.1007/978-3-030-88494-9_3</a>.","apa":"Lukina, A., Schilling, C., &#38; Henzinger, T. A. (2021). Into the unknown: active monitoring of neural networks. In <i>21st International Conference on Runtime Verification</i> (Vol. 12974, pp. 42–61). Cham: Springer Nature. <a href=\"https://doi.org/10.1007/978-3-030-88494-9_3\">https://doi.org/10.1007/978-3-030-88494-9_3</a>","mla":"Lukina, Anna, et al. “Into the Unknown: Active Monitoring of Neural Networks.” <i>21st International Conference on Runtime Verification</i>, vol. 12974, Springer Nature, 2021, pp. 42–61, doi:<a href=\"https://doi.org/10.1007/978-3-030-88494-9_3\">10.1007/978-3-030-88494-9_3</a>."},"place":"Cham","user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","oa":1,"language":[{"iso":"eng"}],"department":[{"_id":"ToHe"}],"month":"10","arxiv":1},{"volume":325,"date_created":"2020-02-21T16:44:03Z","day":"24","author":[{"orcid":"0000-0002-2985-7724","first_name":"Thomas A","id":"40876CD8-F248-11E8-B48F-1D18A9856A87","full_name":"Henzinger, Thomas A","last_name":"Henzinger"},{"last_name":"Lukina","full_name":"Lukina, Anna","id":"CBA4D1A8-0FE8-11E9-BDE6-07BFE5697425","first_name":"Anna"},{"first_name":"Christian","orcid":"0000-0003-3658-1065","last_name":"Schilling","id":"3A2F4DCE-F248-11E8-B48F-1D18A9856A87","full_name":"Schilling, Christian"}],"oa_version":"Published Version","title":"Outside the box: Abstraction-based monitoring of neural networks","file_date_updated":"2020-09-21T07:12:32Z","publication_status":"published","has_accepted_license":"1","tmp":{"name":"Creative Commons Attribution-NonCommercial 4.0 International (CC BY-NC 4.0)","legal_code_url":"https://creativecommons.org/licenses/by-nc/4.0/legalcode","image":"/images/cc_by_nc.png","short":"CC BY-NC (4.0)"},"abstract":[{"lang":"eng","text":"Neural networks have demonstrated unmatched performance in a range of classification tasks. Despite numerous efforts of the research community, novelty detection remains one of the significant limitations of neural networks. The ability to identify previously unseen inputs as novel is crucial for our understanding of the decisions made by neural networks. At runtime, inputs not falling into any of the categories learned during training cannot be classified correctly by the neural network. Existing approaches treat the neural network as a black box and try to detect novel inputs based on the confidence of the output predictions. However, neural networks are not trained to reduce their confidence for novel inputs, which limits the effectiveness of these approaches. We propose a framework to monitor a neural network by observing the hidden layers. We employ a common abstraction from program analysis - boxes - to identify novel behaviors in the monitored layers, i.e., inputs that cause behaviors outside the box. For each neuron, the boxes range over the values seen in training. The framework is efficient and flexible to achieve a desired trade-off between raising false warnings and detecting novel inputs. We illustrate the performance and the robustness to variability in the unknown classes on popular image-classification benchmarks."}],"intvolume":"       325","department":[{"_id":"ToHe"}],"file":[{"file_id":"8540","date_updated":"2020-09-21T07:12:32Z","creator":"dernst","file_size":1692214,"date_created":"2020-09-21T07:12:32Z","checksum":"80642fa0b6cd7da95dcd87d63789ad5e","relation":"main_file","access_level":"open_access","content_type":"application/pdf","success":1,"file_name":"2020_ECAI_Henzinger.pdf"}],"arxiv":1,"month":"02","citation":{"chicago":"Henzinger, Thomas A, Anna Lukina, and Christian Schilling. “Outside the Box: Abstraction-Based Monitoring of Neural Networks.” In <i>24th European Conference on Artificial Intelligence</i>, 325:2433–40. IOS Press, 2020. <a href=\"https://doi.org/10.3233/FAIA200375\">https://doi.org/10.3233/FAIA200375</a>.","ista":"Henzinger TA, Lukina A, Schilling C. 2020. Outside the box: Abstraction-based monitoring of neural networks. 24th European Conference on Artificial Intelligence. ECAI: European Conference on Artificial Intelligence, Frontiers in Artificial Intelligence and Applications, vol. 325, 2433–2440.","apa":"Henzinger, T. A., Lukina, A., &#38; Schilling, C. (2020). Outside the box: Abstraction-based monitoring of neural networks. In <i>24th European Conference on Artificial Intelligence</i> (Vol. 325, pp. 2433–2440). Santiago de Compostela, Spain: IOS Press. <a href=\"https://doi.org/10.3233/FAIA200375\">https://doi.org/10.3233/FAIA200375</a>","mla":"Henzinger, Thomas A., et al. “Outside the Box: Abstraction-Based Monitoring of Neural Networks.” <i>24th European Conference on Artificial Intelligence</i>, vol. 325, IOS Press, 2020, pp. 2433–40, doi:<a href=\"https://doi.org/10.3233/FAIA200375\">10.3233/FAIA200375</a>.","ama":"Henzinger TA, Lukina A, Schilling C. Outside the box: Abstraction-based monitoring of neural networks. In: <i>24th European Conference on Artificial Intelligence</i>. Vol 325. IOS Press; 2020:2433-2440. doi:<a href=\"https://doi.org/10.3233/FAIA200375\">10.3233/FAIA200375</a>","ieee":"T. A. Henzinger, A. Lukina, and C. Schilling, “Outside the box: Abstraction-based monitoring of neural networks,” in <i>24th European Conference on Artificial Intelligence</i>, Santiago de Compostela, Spain, 2020, vol. 325, pp. 2433–2440.","short":"T.A. Henzinger, A. Lukina, C. Schilling, in:, 24th European Conference on Artificial Intelligence, IOS Press, 2020, pp. 2433–2440."},"user_id":"4359f0d1-fa6c-11eb-b949-802e58b17ae8","oa":1,"language":[{"iso":"eng"}],"_id":"7505","date_updated":"2023-08-18T06:38:16Z","type":"conference","article_processing_charge":"No","alternative_title":["Frontiers in Artificial Intelligence and Applications"],"doi":"10.3233/FAIA200375","publisher":"IOS Press","quality_controlled":"1","page":"2433-2440","ddc":["000"],"isi":1,"year":"2020","external_id":{"isi":["000650971303002"],"arxiv":["1911.09032"]},"ec_funded":1,"acknowledgement":"We thank Christoph Lampert and Nikolaus Mayer for fruitful discussions. This research was supported in part by the Austrian Science Fund (FWF) under grants S11402-N23 (RiSE/SHiNE) and Z211-N23 (Wittgenstein Award) and the European Union’s Horizon 2020 research and innovation programme under the Marie SkłodowskaCurie grant agreement No. 754411.","conference":{"name":"ECAI: European Conference on Artificial Intelligence","end_date":"2020-09-08","start_date":"2020-08-29","location":"Santiago de Compostela, Spain"},"date_published":"2020-02-24T00:00:00Z","publication":"24th European Conference on Artificial Intelligence","status":"public","project":[{"call_identifier":"H2020","name":"ISTplus - Postdoctoral Fellowships","grant_number":"754411","_id":"260C2330-B435-11E9-9278-68D0E5697425"},{"_id":"25832EC2-B435-11E9-9278-68D0E5697425","call_identifier":"FWF","name":"Rigorous Systems Engineering","grant_number":"S 11407_N23"},{"_id":"25F42A32-B435-11E9-9278-68D0E5697425","grant_number":"Z211","name":"The Wittgenstein Prize","call_identifier":"FWF"}]},{"oa":1,"language":[{"iso":"eng"}],"citation":{"ama":"Alamdari PA, Avni G, Henzinger TA, Lukina A. Formal methods with a touch of magic. In: <i>Proceedings of the 20th Conference on Formal Methods in Computer-Aided Design</i>. TU Wien Academic Press; 2020:138-147. doi:<a href=\"https://doi.org/10.34727/2020/isbn.978-3-85448-042-6_21\">10.34727/2020/isbn.978-3-85448-042-6_21</a>","ieee":"P. A. Alamdari, G. Avni, T. A. Henzinger, and A. Lukina, “Formal methods with a touch of magic,” in <i>Proceedings of the 20th Conference on Formal Methods in Computer-Aided Design</i>, Online Conference, 2020, pp. 138–147.","short":"P.A. Alamdari, G. Avni, T.A. Henzinger, A. Lukina, in:, Proceedings of the 20th Conference on Formal Methods in Computer-Aided Design, TU Wien Academic Press, 2020, pp. 138–147.","chicago":"Alamdari, Par Alizadeh, Guy Avni, Thomas A Henzinger, and Anna Lukina. “Formal Methods with a Touch of Magic.” In <i>Proceedings of the 20th Conference on Formal Methods in Computer-Aided Design</i>, 138–47. TU Wien Academic Press, 2020. <a href=\"https://doi.org/10.34727/2020/isbn.978-3-85448-042-6_21\">https://doi.org/10.34727/2020/isbn.978-3-85448-042-6_21</a>.","ista":"Alamdari PA, Avni G, Henzinger TA, Lukina A. 2020. Formal methods with a touch of magic. Proceedings of the 20th Conference on Formal Methods in Computer-Aided Design.  FMCAD: Formal Methods in Computer-Aided Design, 138–147.","apa":"Alamdari, P. A., Avni, G., Henzinger, T. A., &#38; Lukina, A. (2020). Formal methods with a touch of magic. In <i>Proceedings of the 20th Conference on Formal Methods in Computer-Aided Design</i> (pp. 138–147). Online Conference: TU Wien Academic Press. <a href=\"https://doi.org/10.34727/2020/isbn.978-3-85448-042-6_21\">https://doi.org/10.34727/2020/isbn.978-3-85448-042-6_21</a>","mla":"Alamdari, Par Alizadeh, et al. “Formal Methods with a Touch of Magic.” <i>Proceedings of the 20th Conference on Formal Methods in Computer-Aided Design</i>, TU Wien Academic Press, 2020, pp. 138–47, doi:<a href=\"https://doi.org/10.34727/2020/isbn.978-3-85448-042-6_21\">10.34727/2020/isbn.978-3-85448-042-6_21</a>."},"user_id":"2DF688A6-F248-11E8-B48F-1D18A9856A87","month":"09","department":[{"_id":"ToHe"}],"file":[{"file_id":"9109","date_created":"2021-02-09T09:39:02Z","file_size":990999,"creator":"dernst","date_updated":"2021-02-09T09:39:02Z","relation":"main_file","checksum":"d616d549a0ade78606b16f8a9540820f","success":1,"file_name":"2020_FMCAD_Alamdari.pdf","access_level":"open_access","content_type":"application/pdf"}],"has_accepted_license":"1","abstract":[{"lang":"eng","text":"Machine learning and formal methods have complimentary benefits and drawbacks. In this work, we address the controller-design problem with a combination of techniques from both fields. The use of black-box neural networks in deep reinforcement learning (deep RL) poses a challenge for such a combination. Instead of reasoning formally about the output of deep RL, which we call the wizard, we extract from it a decision-tree based model, which we refer to as the magic book. Using the extracted model as an intermediary, we are able to handle problems that are infeasible for either deep RL or formal methods by themselves. First, we suggest, for the first time, a synthesis procedure that is based on a magic book. We synthesize a stand-alone correct-by-design controller that enjoys the favorable performance of RL. Second, we incorporate a magic book in a bounded model checking (BMC) procedure. BMC allows us to find numerous traces of the plant under the control of the wizard, which a user can use to increase the trustworthiness of the wizard and direct further training."}],"tmp":{"name":"Creative Commons Attribution 4.0 International Public License (CC-BY 4.0)","legal_code_url":"https://creativecommons.org/licenses/by/4.0/legalcode","image":"/images/cc_by.png","short":"CC BY (4.0)"},"publication_status":"published","publication_identifier":{"eissn":["2708-7824"],"isbn":["9783854480426"]},"file_date_updated":"2021-02-09T09:39:02Z","author":[{"first_name":"Par Alizadeh","full_name":"Alamdari, Par Alizadeh","last_name":"Alamdari"},{"id":"463C8BC2-F248-11E8-B48F-1D18A9856A87","full_name":"Avni, Guy","last_name":"Avni","orcid":"0000-0001-5588-8287","first_name":"Guy"},{"first_name":"Thomas A","orcid":"0000-0002-2985-7724","id":"40876CD8-F248-11E8-B48F-1D18A9856A87","full_name":"Henzinger, Thomas A","last_name":"Henzinger"},{"last_name":"Lukina","full_name":"Lukina, Anna","id":"CBA4D1A8-0FE8-11E9-BDE6-07BFE5697425","first_name":"Anna"}],"scopus_import":"1","day":"21","title":"Formal methods with a touch of magic","oa_version":"Published Version","date_created":"2021-01-24T23:01:10Z","project":[{"grant_number":"Z211","name":"The Wittgenstein Prize","call_identifier":"FWF","_id":"25F42A32-B435-11E9-9278-68D0E5697425"}],"publication":"Proceedings of the 20th Conference on Formal Methods in Computer-Aided Design","status":"public","acknowledgement":"This research was supported in part by the Austrian Science Fund (FWF) under grant Z211-N23 (Wittgenstein Award).","conference":{"end_date":"2020-09-24","start_date":"2020-09-21","name":" FMCAD: Formal Methods in Computer-Aided Design","location":"Online Conference"},"date_published":"2020-09-21T00:00:00Z","year":"2020","page":"138-147","ddc":["000"],"quality_controlled":"1","doi":"10.34727/2020/isbn.978-3-85448-042-6_21","article_processing_charge":"No","publisher":"TU Wien Academic Press","date_updated":"2021-02-09T09:39:59Z","_id":"9040","type":"conference"}]