Xamarin Forms: Selected items get cleared when perform search in listview

I have done the fetching of contacts from the phone using this blog.

Now I am trying to add the selection of contacts. Using a switch I have done the selection. But the selected contacts are clearing when performing a search operation.

xaml

<Switch
    Toggled="OnToggledEvent"
    HorizontalOptions="EndAndExpand"
    VerticalOptions="CenterAndExpand"/>

xaml.cs

public List<Contact> contactList;
public MainPage(IContactsService contactService)
{
    InitializeComponent();
    contactList = new List<Contact>();
    BindingContext = new ContactsViewModel(contactService);
}

void OnToggledEvent(object sender, EventArgs args)
{
    ViewCell cell = (sender as Xamarin.Forms.Switch).Parent.Parent as ViewCell;
    if (cell.BindingContext is Contact)
    {
        Contact contact = cell.BindingContext as Contact;
        if (contact != null)
        {
            if (contact != null && !contactList.Contains(contact))
            {
                contactList.Add(contact);
            }
            else if (contact != null && contactList.Contains(contact))
            {
                contactList.Remove(contact);
            }
        }
    }
    Debug.WriteLine("contactList:>>" + contactList.Count);
}

ContactsViewModel

public class ContactsViewModel : INotifyPropertyChanged
{
    IContactsService _contactService;

    public event PropertyChangedEventHandler PropertyChanged;
    protected virtual void OnPropertyChanged([CallerMemberName] string propertyName = null)
    {
        PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
    }
    public string Title => "Contacts";


    string search;
    public string SearchText
    {

        get { return search; }

        set
        {

            if (search != value)
            {
                search = value;
                OnPropertyChanged("SearchText");

                if (string.IsNullOrEmpty(SearchText))
                {
                    FilteredContacts = new ObservableCollection<Contact>(Contacts);

                }

                else
                {

                    FilteredContacts = new ObservableCollection<Contact>(Contacts?.ToList()?.Where(s => !string.IsNullOrEmpty(s.Name) && s.Name.ToLower().Contains(SearchText.ToLower())));

                }
            }

        }

    }

    public ObservableCollection<Contact> Contacts { get; set; }

    ObservableCollection<Contact> filteredContacts;
    public ObservableCollection<Contact> FilteredContacts
    {
        get { return filteredContacts; }

        set
        {

            if (filteredContacts != value)
            {
                filteredContacts = value;
                OnPropertyChanged("FilteredContacts");
            }
        }
    }
    public ContactsViewModel(IContactsService contactService)
    {

        _contactService = contactService;
        Contacts = new ObservableCollection<Contact>();
        Xamarin.Forms.BindingBase.EnableCollectionSynchronization(Contacts, null, ObservableCollectionCallback);
        _contactService.OnContactLoaded += OnContactLoaded;
        LoadContacts();
        FilteredContacts = Contacts;
    }


    void ObservableCollectionCallback(IEnumerable collection, object context, Action accessMethod, bool writeAccess)
    {
        // `lock` ensures that only one thread access the collection at a time
        lock (collection)
        {
            accessMethod?.Invoke();
        }
    }

    private void OnContactLoaded(object sender, ContactEventArgs e)
    {
        Contacts.Add(e.Contact);
    }
    async Task LoadContacts()
    {
        try
        {
            await _contactService.RetrieveContactsAsync();
        }
        catch (TaskCanceledException)
        {
            Console.WriteLine("Task was cancelled");
        }
    }
}

I am adding the selected contact to a list when toggling the switch. If again click the switch I will remove the contact from the list. But the problem is when searching for a contact, already selected contacts get clear. I try to fix this using IsToggled property of switch, but no luck.

I have added a sample project here for the reference.