@inproceedings {grishina2023:earlybird,
	title = {The EarlyBIRD Catches the Bug: On Exploiting Early Layers of Encoder Models for More Efficient Code Classification},
	journal = {ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering (ESEC/FSE 2023)},
	year = {2023},
	month = {12/2023},
	publisher = {ACM},
	abstract = {The use of modern Natural Language Processing (NLP) techniques has shown to be beneficial for software engineering tasks, such as vulnerability detection and type inference. However, training deep NLP models requires significant computational resources. This paper explores techniques that aim at achieving the best usage of resources and available information in these models.We propose a generic approach, EarlyBIRD, to build composite representations of code from the early layers of a pre-trained transformer model. We empirically investigate the viability of this approach on the CodeBERT model by comparing the performance of 12 strategies for creating composite representations with the standard practice of only using the last encoder layer.Our evaluation on four datasets shows that several early layer combinations yield better performance on defect detection, and some combinations improve multi-class classification. More specifically, we obtain a +2 average improvement of detection accuracy on Devign with only 3 out of 12 layers of CodeBERT and a 3.3x speed-up of fine-tuning. These findings show that early layers can be used to obtain better results using the same resources, as well as to reduce resource usage during fine-tuning and inference.},
	keywords = {ai4code, AI4SE, code classification, ML4SE, model optimization, sustainability, transformer, vulnerability detection},
	author = {Grishina, Anastasiia and Hort, Max and Moonen, Leon}
}
@techreport {grishina2023:earlybird:arxiv,
	title = {The EarlyBIRD Catches the Bug: On Exploiting Early Layers of Encoder Models for More Efficient Code Classification},
	year = {2023},
	month = {05/2023},
	publisher = {arXiv},
	abstract = {The use of modern Natural Language Processing (NLP) techniques has shown to be beneficial for software engineering tasks, such as vulnerability detection and type inference. However, training deep NLP models requires significant computational resources. This paper explores techniques that aim at achieving the best usage of resources and available information in these models.We propose a generic approach, EarlyBIRD, to build composite representations of code from the early layers of a pre-trained transformer model. We empirically investigate the viability of this approach on the CodeBERT model by comparing the performance of 12 strategies for creating composite representations with the standard practice of only using the last encoder layer.Our evaluation on four datasets shows that several early layer combinations yield better performance on defect detection, and some combinations improve multi-class classification. More specifically, we obtain a +2 average improvement of detection accuracy on Devign with only 3 out of 12 layers of CodeBERT and a 3.3x speed-up of fine-tuning. These findings show that early layers can be used to obtain better results using the same resources, as well as to reduce resource usage during fine-tuning and inference.},
	keywords = {ai4code, AI4SE, code classification, ML4SE, model optimization, sustainability, transformer, vulnerability detection},
	doi = {10.48550/arXiv.2305.04940},
	url = {https://arxiv.org/abs/2305.04940},
	author = {Grishina, Anastasiia and Hort, Max and Moonen, Leon}
}
@inproceedings {liventsev2023:fully,
	title = {Fully Autonomous Programming with Large Language Models},
	journal = {ACM SIGEVO Genetic and Evolutionary Computation Conference (GECCO 2023)},
	year = {2023},
	month = {07/2023},
	publisher = {ACM},
	abstract = {Current approaches to program synthesis with Large Language Models (LLMs) exhibit a "near miss syndrome": they tend to generate programs that semantically resemble the correct answer (as measured by text similarity metrics or human evaluation), but achieve a low or even zero accuracy as measured by unit tests due to small imperfections, such as the wrong input or output format. This calls for an approach known as Synthesize, Execute, Debug (SED), whereby a draft of the solution is generated first, followed by a program repair phase addressing the failed tests. To effectively apply this approach to instruction-driven LLMs, one needs to determine which prompts perform best as instructions for LLMs, as well as strike a balance between repairing unsuccessful programs and replacing them with newly generated ones. We explore these trade-offs empirically, comparing replace-focused, repair-focused, and hybrid debug strategies, as well as different template-based and model-based prompt-generation techniques. We use OpenAI Codex as the LLM and Program Synthesis Benchmark 2 as a database of problem descriptions and tests for evaluation. The resulting framework outperforms both conventional usage of Codex without the repair phase and traditional genetic programming approaches.},
	keywords = {automated program repair, automatic programming, large language models},
	doi = {10.1145/3583131.3590481},
	author = {Liventsev, Vadim and Grishina, Anastasiia and H{\"a}rm{\"a}, Aki and Moonen, Leon}
}
@techreport {liventsev2023:fully:arxiv,
	title = {Fully Autonomous Programming with Large Language Models},
	year = {2023},
	month = {04/2023},
	publisher = {arXiv},
	abstract = {Current approaches to program synthesis with Large Language Models (LLMs) exhibit a "near miss syndrome": they tend to generate programs that semantically resemble the correct answer (as measured by text similarity metrics or human evaluation), but achieve a low or even zero accuracy as measured by unit tests due to small imperfections, such as the wrong input or output format. This calls for an approach known as Synthesize, Execute, Debug (SED), whereby a draft of the solution is generated first, followed by a program repair phase addressing the failed tests. To effectively apply this approach to instruction-driven LLMs, one needs to determine which prompts perform best as instructions for LLMs, as well as strike a balance between repairing unsuccessful programs and replacing them with newly generated ones. We explore these trade-offs empirically, comparing replace-focused, repair-focused, and hybrid debug strategies, as well as different template-based and model-based prompt-generation techniques. We use OpenAI Codex as the LLM and Program Synthesis Benchmark 2 as a database of problem descriptions and tests for evaluation. The resulting framework outperforms both conventional usage of Codex without the repair phase and traditional genetic programming approaches.},
	keywords = {automated program repair, automatic programming, large language models},
	doi = {10.1145/3583131.3590481},
	url = {https://arxiv.org/abs/2304.10423},
	author = {Liventsev, Vadim and Grishina, Anastasiia and H{\"a}rm{\"a}, Aki and Moonen, Leon}
}
@techreport {grishina2022:ds-arxiv,
	title = {Enabling Automatic Repair of Source Code Vulnerabilities Using Data-Driven Methods},
	year = {2022},
	note = {Accepted for the ICSE {\textquoteright}22 Doctoral Symposium},
	month = {02/2022},
	pages = {3 pages},
	publisher = {arXiv},
	abstract = {Users around the world rely on software-intensive systems in their day-to-day activities. These systems regularly contain bugs and security vulnerabilities. To facilitate bug fixing, data-driven models of automatic program repair use pairs of buggy and fixed code to learn transformations that fix errors in code. However, automatic repair of security vulnerabilities remains under-explored. In this work, we propose ways to improve code representations for vulnerability repair from three perspectives: input data type, data-driven models, and downstream tasks. The expected results of this work are improved code representations for automatic program repair and, specifically, fixing security vulnerabilities.},
	keywords = {automatic program repair, graph-based machine learning, ml4code, natural language processing, software security, static analysis},
	url = {http://arxiv.org/abs/2202.03055},
	author = {Grishina, Anastasiia}
}
@inproceedings {42490,
	title = {Enabling Automatic Repair of Source Code Vulnerabilities Using Data-Driven Methods},
	journal = {44th International Conference on Software Engineering Companion (ICSE {\textquoteright}22 Companion), Doctoral Symposium},
	year = {2022},
	month = {05/2022},
	pages = {275-277},
	publisher = {Association for Computing Machinery},
	abstract = {	
Users around the world rely on software-intensive systems in their day-to-day activities. These systems regularly contain bugs and security vulnerabilities. To facilitate bug fixing, data-driven models of automatic program repair use pairs of buggy and fixed code to learn transformations that fix errors in code. However, automatic repair of security vulnerabilities remains under-explored. In this work, we propose ways to improve code representations for vulnerability repair from three perspectives: input data type, data-driven models, and downstream tasks. The expected results of this work are improved code representations for automatic program repair and, specifically, fixing security vulnerabilities.},
	keywords = {automatic program repair, graph-based machine learning, ml4code, natural language processing, software security, static analysis},
	isbn = {978-1-4503-9223-5/22/05},
	doi = {10.1145/3510454.3517063},
	author = {Grishina, Anastasiia}
}
@techreport {bataineh:2021:integrate:arxiv,
	title = {Towards More Reliable Automated Program Repair by Integrating Static Analysis Techniques},
	year = {2021},
	month = {11/2021},
	pages = {1-10},
	publisher = {arXiv},
	abstract = {A long-standing open challenge for automated program repair is the overfitting problem, which is caused by having insufficient or incomplete specifications to validate whether a generated patch is correct or not. Most available repair systems rely on weak specifications (i.e., specifications that are synthesized from test cases) which limits the quality of generated repairs. To strengthen specifications and improve the quality of repairs, we propose to closer integrate static bug detection techniques with automated program repair. The integration combines automated program repair with static analysis techniques in such a way that bug detection patterns can be synthesized into specifications that the repair system can use. We explore the feasibility of such integration using two types of bugs: arithmetic bugs, such as integer overflow, and logical bugs, such as termination bugs. As part of our analysis, we make several observations that help to improve patch generation for these classes of bugs. Moreover, these observations assist with narrowing down the candidate patch search space, and inferring an effective search order.},
	keywords = {automated program repair, bug detection, conditional mutation, integer overflow, non-termination, static analysis},
	doi = {arXiv:2111.05713},
	url = {https://arxiv.org/abs/2111.05713},
	author = {Omar I. {Al-Bataineh} and Grishina, Anastasiia and Moonen, Leon}
}
@inproceedings {bataineh:2021:integrate,
	title = {Towards More Reliable Automated Program Repair by Integrating Static Analysis Techniques},
	journal = {21st IEEE International Conference on Software Quality, Reliability, and Security},
	year = {2021},
	month = {12/2021},
	publisher = { IEEE},
	abstract = {A long-standing open challenge for automated program repair is the overfitting problem, which is caused by having insufficient or incomplete specifications to validate whether a generated patch is correct or not. Most available repair systems rely on weak specifications (i.e., specifications that are synthesized from test cases) which limits the quality of generated repairs. To strengthen specifications and improve the quality of repairs, we propose to closer integrate static bug detection techniques with automated program repair. The integration combines automated program repair with static analysis techniques in such a way that bug detection patterns can be synthesized into specifications that the repair system can use. We explore the feasibility of such integration using two types of bugs: arithmetic bugs, such as integer overflow, and logical bugs, such as termination bugs. As part of our analysis, we make several observations that help to improve patch generation for these classes of bugs. Moreover, these observations assist with narrowing down the candidate patch search space, and inferring an effective search order.},
	keywords = {automated program repair, bug detection, conditional mutation, integer overflow, non-termination, static analysis},
	author = {Omar I. {Al-Bataineh} and Grishina, Anastasiia and Moonen, Leon}
}
